2023-03-27T10:03:50.333 INFO:root:teuthology version: 0.0.1.dev1+ge372916 2023-03-27T10:03:50.334 DEBUG:teuthology.run:Teuthology command: teuthology --name rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi --owner scheduled_rishabh@teuthology --verbose --description fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/secure wsync/no} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{export-check n/5 replication/always} standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/yes 5-workunit/suites/ffsb}} --archive /home/teuthworker/archive/rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi/7221317 -- /home/teuthworker/archive/rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi/7221317/orig.config.yaml 2023-03-27T10:03:50.393 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/ 2023-03-27T10:03:50.477 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi/7221317 branch: wip-rishabh-cephman-6 description: fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/secure wsync/no} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{export-check n/5 replication/always} standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/no 3-snaps/yes 4-flush/yes 5-workunit/suites/ffsb}} email: null first_in_suite: false job_id: '7221317' kernel: client: sha1: distro flavor: default kdb: true sha1: 1776e8f494e94fd3414d7679e5782ec22e83c95f last_in_suite: false log-rotate: ceph-mds: 10G ceph-osd: 10G machine_type: smithi name: rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 20 - machine: disk: 200 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: wip-rishabh-cephman-6 ceph: cephfs: ec_profile: - m=2 - k=2 - crush-failure-domain=osd max_mds: 5 session_timeout: 300 standby_replay: true conf: client: client mount timeout: 600 debug client: 20 debug ms: 1 rados mon op timeout: 900 rados osd op timeout: 900 global: ms die on skipped message: false mds: debug mds: 20 debug mds balancer: 20 debug ms: 1 mds bal fragment size max: 10000 mds bal merge size: 5 mds bal split bits: 3 mds bal split size: 100 mds debug frag: true mds debug scatterstat: true mds op complaint time: 180 mds verify scatter: true mds_bal_replicate_threshold: 1 osd op complaint time: 180 rados mon op timeout: 900 rados osd op timeout: 900 mgr: cephsqlite lock renewal timeout: 900000 debug cephsqlite: 20 debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon op complaint time: 120 osd: bluestore block size: 96636764160 bluestore compression mode: aggressive bluestore fsck on mount: true debug bluefs: 20 debug bluestore: 20 debug ms: 1 debug osd: 20 debug rocksdb: 10 filestore flush min: 0 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd heartbeat grace: 60 osd objectstore: bluestore osd op complaint time: 180 osd shutdown pgref assert: true osd_max_omap_entries_per_request: 10 flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - overall HEALTH_ - \(FS_DEGRADED\) - \(MDS_FAILED\) - \(MDS_DEGRADED\) - \(FS_WITH_FAILED_MDS\) - \(MDS_DAMAGE\) - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(FS_INLINE_DATA_DEPRECATED\) - overall HEALTH_ - \(OSD_DOWN\) - \(OSD_ - but it is still running - is not responding - SLOW_OPS - slow request sha1: b70fdf973c2ffa23c0d8aa6918519da05d922302 subvols: create: 2 subvol_options: --namespace-isolated --size 25000000000 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 ceph-fuse: client.0: mount_subvol_num: 0 check-counter: counters: mds: - mds.exported - mds.imported - min: 3 name: mds_cache.dir_update - min: 3 name: mds_cache.dir_update_receipt - mds.root_rsnaps - mds_server.req_mksnap_latency.avgcount - mds_server.req_rmsnap_latency.avgcount - mds.dir_split install: ceph: flavor: default sha1: b70fdf973c2ffa23c0d8aa6918519da05d922302 kclient: client.0: mount_subvol_num: 1 mntopts: - ms_mode=secure - nowsync syntax: v1 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:logrotate_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: CephManager-in-CephFSTestCase sha1: 342392241eddd9827618f3dbb8134df7441718a8 owner: scheduled_rishabh@teuthology priority: 200 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mgr.x - mds.a - mds.d - mds.g - mds.j - osd.0 - osd.3 - osd.6 - osd.9 - client.0 - - mon.b - mgr.y - mds.b - mds.e - mds.h - mds.k - osd.1 - osd.4 - osd.7 - osd.10 - - mon.c - mgr.z - mds.c - mds.f - mds.i - mds.l - osd.2 - osd.5 - osd.8 - osd.11 sha1: b70fdf973c2ffa23c0d8aa6918519da05d922302 sleep_before_teardown: 0 suite: fs suite_branch: CephManager-in-CephFSTestCase suite_path: /home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa suite_relpath: qa suite_repo: https://github.com/rishabh-d-dave/ceph suite_sha1: 342392241eddd9827618f3dbb8134df7441718a8 syslog: ignorelist: - WARNING*.*check_session_state - WARNING*.*__ceph_remove_cap targets: smithi159.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac= smithi161.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi171.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:rhel8 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: extra_packages: deb: - python3-cephfs - cephfs-shell - cephfs-top - cephfs-mirror rpm: - python3-cephfs - cephfs-top - cephfs-mirror extra_system_packages: deb: - bison - flex - libelf-dev - libssl-dev - network-manager - iproute2 - util-linux - dump - indent - libaio-dev - libtool-bin - uuid-dev - xfslibs-dev - postgresql - postgresql-client - postgresql-common - postgresql-contrib rpm: - bison - flex - elfutils-libelf-devel - openssl-devel - NetworkManager - iproute - util-linux - libacl-devel - libaio-devel - libattr-devel - libtool - libuuid-devel - xfsdump - xfsprogs - xfsprogs-devel - libaio-devel - libtool - libuuid-devel - xfsprogs-devel - postgresql - postgresql-server - postgresql-contrib - cephadm: roleless: false - cephadm.shell: mon.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - cephadm.shell: mon.a: - ceph fs dump - ceph osd dump - fs.ready: timeout: 300 - kclient: null - check-counter: {} - exec: mon.a: - ceph mgr module enable snap_schedule - ceph config set mgr mgr/snap_schedule/allow_m_granularity true - ceph config set mgr mgr/snap_schedule/dump_on_update true - ceph fs snap-schedule add --fs=cephfs --path=/ --snap_schedule=1M - ceph fs snap-schedule retention add --fs=cephfs --path=/ --retention-spec-or-period=6M3h - ceph fs snap-schedule status --fs=cephfs --path=/ - ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true - date +%s > START_TIME - full_sequential_finally: - exec: mon.a: - date +%s > END_TIME - START_TIME=$(cat START_TIME); END_TIME=$(cat END_TIME); DIFF_TIME=$((600-(END_TIME-START_TIME))); if [ "$DIFF_TIME" -gt 0 ]; then sleep "$DIFF_TIME"; fi - ceph fs snap-schedule status --fs=cephfs --path=/ - ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true - background_exec: mon.a: - while sleep 13; do ceph tell mds.cephfs:0 flush journal; done - workunit: clients: all: - suites/ffsb.sh teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: e3729161eecf8e6d09a4069acbb883cdcd5e9f50 timestamp: 2023-03-27_06:39:30 tube: smithi user: rishabh verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.4538 2023-03-27T10:03:50.478 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa; will attempt to use it 2023-03-27T10:03:50.479 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks 2023-03-27T10:03:50.479 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-03-27T10:03:50.481 INFO:teuthology.task.internal:Checking packages... 2023-03-27T10:03:50.509 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'b70fdf973c2ffa23c0d8aa6918519da05d922302' 2023-03-27T10:03:50.510 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-03-27T10:03:50.511 INFO:teuthology.packaging:ref: None 2023-03-27T10:03:50.511 INFO:teuthology.packaging:tag: None 2023-03-27T10:03:50.512 INFO:teuthology.packaging:branch: wip-rishabh-cephman-6 2023-03-27T10:03:50.512 INFO:teuthology.packaging:sha1: b70fdf973c2ffa23c0d8aa6918519da05d922302 2023-03-27T10:03:50.512 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=wip-rishabh-cephman-6 2023-03-27T10:03:50.673 INFO:teuthology.task.internal:Found packages for ceph version 18.0.0-2533.gb70fdf97 2023-03-27T10:03:50.673 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-03-27T10:03:50.686 INFO:teuthology.task.internal:no buildpackages task found 2023-03-27T10:03:50.686 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-03-27T10:03:50.722 INFO:teuthology.task.internal:Saving configuration 2023-03-27T10:03:50.744 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-03-27T10:03:50.755 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-03-27T10:03:50.785 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi159.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi/7221317', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-03-27 09:58:45.891622', 'locked_by': 'scheduled_rishabh@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-03-27T10:03:50.809 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi161.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi/7221317', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-03-27 09:58:45.889577', 'locked_by': 'scheduled_rishabh@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-03-27T10:03:50.832 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi171.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi/7221317', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-03-27 09:58:45.892856', 'locked_by': 'scheduled_rishabh@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-03-27T10:03:50.833 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-03-27T10:03:50.840 INFO:teuthology.task.internal:roles: ubuntu@smithi159.front.sepia.ceph.com - ['mon.a', 'mgr.x', 'mds.a', 'mds.d', 'mds.g', 'mds.j', 'osd.0', 'osd.3', 'osd.6', 'osd.9', 'client.0'] 2023-03-27T10:03:50.840 INFO:teuthology.task.internal:roles: ubuntu@smithi161.front.sepia.ceph.com - ['mon.b', 'mgr.y', 'mds.b', 'mds.e', 'mds.h', 'mds.k', 'osd.1', 'osd.4', 'osd.7', 'osd.10'] 2023-03-27T10:03:50.840 INFO:teuthology.task.internal:roles: ubuntu@smithi171.front.sepia.ceph.com - ['mon.c', 'mgr.z', 'mds.c', 'mds.f', 'mds.i', 'mds.l', 'osd.2', 'osd.5', 'osd.8', 'osd.11'] 2023-03-27T10:03:50.840 INFO:teuthology.run_tasks:Running task console_log... 2023-03-27T10:03:50.997 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7ff3a58b15e0>, signals=[15]) 2023-03-27T10:03:50.998 INFO:teuthology.run_tasks:Running task internal.connect... 2023-03-27T10:03:51.006 INFO:teuthology.task.internal:Opening connections... 2023-03-27T10:03:51.007 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi159.front.sepia.ceph.com 2023-03-27T10:03:51.009 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-03-27T10:03:51.080 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi161.front.sepia.ceph.com 2023-03-27T10:03:51.082 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi161.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-03-27T10:03:51.147 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi171.front.sepia.ceph.com 2023-03-27T10:03:51.148 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi171.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-03-27T10:03:51.216 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-03-27T10:03:51.228 DEBUG:teuthology.orchestra.run.smithi159:> uname -m 2023-03-27T10:03:51.250 INFO:teuthology.orchestra.run.smithi159.stdout:x86_64 2023-03-27T10:03:51.250 DEBUG:teuthology.orchestra.run.smithi159:> cat /etc/os-release 2023-03-27T10:03:51.309 INFO:teuthology.orchestra.run.smithi159.stdout:NAME="Red Hat Enterprise Linux" 2023-03-27T10:03:51.309 INFO:teuthology.orchestra.run.smithi159.stdout:VERSION="8.6 (Ootpa)" 2023-03-27T10:03:51.310 INFO:teuthology.orchestra.run.smithi159.stdout:ID="rhel" 2023-03-27T10:03:51.310 INFO:teuthology.orchestra.run.smithi159.stdout:ID_LIKE="fedora" 2023-03-27T10:03:51.310 INFO:teuthology.orchestra.run.smithi159.stdout:VERSION_ID="8.6" 2023-03-27T10:03:51.310 INFO:teuthology.orchestra.run.smithi159.stdout:PLATFORM_ID="platform:el8" 2023-03-27T10:03:51.310 INFO:teuthology.orchestra.run.smithi159.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-03-27T10:03:51.310 INFO:teuthology.orchestra.run.smithi159.stdout:ANSI_COLOR="0;31" 2023-03-27T10:03:51.311 INFO:teuthology.orchestra.run.smithi159.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-03-27T10:03:51.311 INFO:teuthology.orchestra.run.smithi159.stdout:HOME_URL="https://www.redhat.com/" 2023-03-27T10:03:51.311 INFO:teuthology.orchestra.run.smithi159.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-03-27T10:03:51.311 INFO:teuthology.orchestra.run.smithi159.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-03-27T10:03:51.311 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:03:51.311 INFO:teuthology.orchestra.run.smithi159.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-03-27T10:03:51.311 INFO:teuthology.orchestra.run.smithi159.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-03-27T10:03:51.311 INFO:teuthology.orchestra.run.smithi159.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-03-27T10:03:51.312 INFO:teuthology.orchestra.run.smithi159.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-03-27T10:03:51.312 INFO:teuthology.lock.ops:Updating smithi159.front.sepia.ceph.com on lock server 2023-03-27T10:03:51.338 DEBUG:teuthology.orchestra.run.smithi161:> uname -m 2023-03-27T10:03:51.358 INFO:teuthology.orchestra.run.smithi161.stdout:x86_64 2023-03-27T10:03:51.359 DEBUG:teuthology.orchestra.run.smithi161:> cat /etc/os-release 2023-03-27T10:03:51.417 INFO:teuthology.orchestra.run.smithi161.stdout:NAME="Red Hat Enterprise Linux" 2023-03-27T10:03:51.418 INFO:teuthology.orchestra.run.smithi161.stdout:VERSION="8.6 (Ootpa)" 2023-03-27T10:03:51.418 INFO:teuthology.orchestra.run.smithi161.stdout:ID="rhel" 2023-03-27T10:03:51.418 INFO:teuthology.orchestra.run.smithi161.stdout:ID_LIKE="fedora" 2023-03-27T10:03:51.418 INFO:teuthology.orchestra.run.smithi161.stdout:VERSION_ID="8.6" 2023-03-27T10:03:51.418 INFO:teuthology.orchestra.run.smithi161.stdout:PLATFORM_ID="platform:el8" 2023-03-27T10:03:51.418 INFO:teuthology.orchestra.run.smithi161.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-03-27T10:03:51.419 INFO:teuthology.orchestra.run.smithi161.stdout:ANSI_COLOR="0;31" 2023-03-27T10:03:51.419 INFO:teuthology.orchestra.run.smithi161.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-03-27T10:03:51.419 INFO:teuthology.orchestra.run.smithi161.stdout:HOME_URL="https://www.redhat.com/" 2023-03-27T10:03:51.419 INFO:teuthology.orchestra.run.smithi161.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-03-27T10:03:51.419 INFO:teuthology.orchestra.run.smithi161.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-03-27T10:03:51.419 INFO:teuthology.orchestra.run.smithi161.stdout: 2023-03-27T10:03:51.419 INFO:teuthology.orchestra.run.smithi161.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-03-27T10:03:51.419 INFO:teuthology.orchestra.run.smithi161.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-03-27T10:03:51.420 INFO:teuthology.orchestra.run.smithi161.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-03-27T10:03:51.420 INFO:teuthology.orchestra.run.smithi161.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-03-27T10:03:51.420 INFO:teuthology.lock.ops:Updating smithi161.front.sepia.ceph.com on lock server 2023-03-27T10:03:51.446 DEBUG:teuthology.orchestra.run.smithi171:> uname -m 2023-03-27T10:03:51.467 INFO:teuthology.orchestra.run.smithi171.stdout:x86_64 2023-03-27T10:03:51.468 DEBUG:teuthology.orchestra.run.smithi171:> cat /etc/os-release 2023-03-27T10:03:51.526 INFO:teuthology.orchestra.run.smithi171.stdout:NAME="Red Hat Enterprise Linux" 2023-03-27T10:03:51.527 INFO:teuthology.orchestra.run.smithi171.stdout:VERSION="8.6 (Ootpa)" 2023-03-27T10:03:51.527 INFO:teuthology.orchestra.run.smithi171.stdout:ID="rhel" 2023-03-27T10:03:51.527 INFO:teuthology.orchestra.run.smithi171.stdout:ID_LIKE="fedora" 2023-03-27T10:03:51.527 INFO:teuthology.orchestra.run.smithi171.stdout:VERSION_ID="8.6" 2023-03-27T10:03:51.527 INFO:teuthology.orchestra.run.smithi171.stdout:PLATFORM_ID="platform:el8" 2023-03-27T10:03:51.528 INFO:teuthology.orchestra.run.smithi171.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-03-27T10:03:51.528 INFO:teuthology.orchestra.run.smithi171.stdout:ANSI_COLOR="0;31" 2023-03-27T10:03:51.528 INFO:teuthology.orchestra.run.smithi171.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-03-27T10:03:51.528 INFO:teuthology.orchestra.run.smithi171.stdout:HOME_URL="https://www.redhat.com/" 2023-03-27T10:03:51.528 INFO:teuthology.orchestra.run.smithi171.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-03-27T10:03:51.528 INFO:teuthology.orchestra.run.smithi171.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-03-27T10:03:51.528 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:03:51.529 INFO:teuthology.orchestra.run.smithi171.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-03-27T10:03:51.529 INFO:teuthology.orchestra.run.smithi171.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-03-27T10:03:51.529 INFO:teuthology.orchestra.run.smithi171.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-03-27T10:03:51.529 INFO:teuthology.orchestra.run.smithi171.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-03-27T10:03:51.530 INFO:teuthology.lock.ops:Updating smithi171.front.sepia.ceph.com on lock server 2023-03-27T10:03:51.555 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-03-27T10:03:51.570 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-03-27T10:03:51.599 INFO:teuthology.task.internal:Checking for old test directory... 2023-03-27T10:03:51.599 DEBUG:teuthology.orchestra.run.smithi159:> test '!' -e /home/ubuntu/cephtest 2023-03-27T10:03:51.602 DEBUG:teuthology.orchestra.run.smithi161:> test '!' -e /home/ubuntu/cephtest 2023-03-27T10:03:51.605 DEBUG:teuthology.orchestra.run.smithi171:> test '!' -e /home/ubuntu/cephtest 2023-03-27T10:03:51.624 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-03-27T10:03:51.630 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-03-27T10:03:51.630 DEBUG:teuthology.orchestra.run.smithi159:> test -z $(ls -A /var/lib/ceph) 2023-03-27T10:03:51.662 DEBUG:teuthology.orchestra.run.smithi161:> test -z $(ls -A /var/lib/ceph) 2023-03-27T10:03:51.665 DEBUG:teuthology.orchestra.run.smithi171:> test -z $(ls -A /var/lib/ceph) 2023-03-27T10:03:51.695 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-03-27T10:03:51.788 INFO:teuthology.run_tasks:Running task kernel... 2023-03-27T10:03:51.807 INFO:teuthology.task.kernel:normalize config orig: {'client': {'sha1': 'distro'}, 'flavor': 'default', 'kdb': True, 'sha1': '1776e8f494e94fd3414d7679e5782ec22e83c95f'} 2023-03-27T10:03:51.808 INFO:teuthology.task.kernel:normalize config final: {'client.0': {'sha1': 'distro'}} 2023-03-27T10:03:51.808 INFO:teuthology.task.kernel:config {'client.0': {'sha1': 'distro'}}, timeout 300 2023-03-27T10:03:51.809 DEBUG:teuthology.orchestra.run.smithi159:> test -f /run/.containerenv -o -f /.dockerenv 2023-03-27T10:03:51.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-03-27T10:03:51.828 DEBUG:teuthology.orchestra.run.smithi159:> uname -r 2023-03-27T10:03:51.887 INFO:teuthology.orchestra.run.smithi159.stdout:4.18.0-372.9.1.el8.x86_64 2023-03-27T10:03:51.887 INFO:teuthology.task.kernel:Running kernel on smithi159: 4.18.0-372.9.1.el8.x86_64 2023-03-27T10:03:51.888 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum install -y kernel 2023-03-27T10:03:53.097 INFO:teuthology.orchestra.run.smithi159.stdout:Updating Subscription Management repositories. 2023-03-27T10:03:53.098 INFO:teuthology.orchestra.run.smithi159.stdout:Unable to read consumer identity 2023-03-27T10:03:54.572 INFO:teuthology.orchestra.run.smithi159.stdout:Copr repo for python3-asyncssh owned by ceph 19 kB/s | 3.5 kB 00:00 2023-03-27T10:03:55.802 INFO:teuthology.orchestra.run.smithi159.stdout:Extra Packages for Enterprise Linux 11 MB/s | 14 MB 00:01 2023-03-27T10:03:58.844 INFO:teuthology.orchestra.run.smithi159.stdout:lab-extras 413 kB/s | 24 kB 00:00 2023-03-27T10:03:59.538 INFO:teuthology.orchestra.run.smithi159.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-03-27T10:03:59.585 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2023-03-27T10:03:59.586 INFO:teuthology.orchestra.run.smithi159.stdout:Nothing to do. 2023-03-27T10:03:59.586 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2023-03-27T10:03:59.719 DEBUG:teuthology.orchestra.run.smithi159:> echo no | sudo yum reinstall kernel || true 2023-03-27T10:04:00.053 INFO:teuthology.orchestra.run.smithi159.stdout:Updating Subscription Management repositories. 2023-03-27T10:04:00.054 INFO:teuthology.orchestra.run.smithi159.stdout:Unable to read consumer identity 2023-03-27T10:04:00.309 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:00:02 ago on Mon 27 Mar 2023 10:03:58 AM UTC. 2023-03-27T10:04:00.349 INFO:teuthology.orchestra.run.smithi159.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-03-27T10:04:00.387 INFO:teuthology.orchestra.run.smithi159.stderr:Error: No packages marked for reinstall. 2023-03-27T10:04:00.433 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum reinstall -y kernel || true 2023-03-27T10:04:00.765 INFO:teuthology.orchestra.run.smithi159.stdout:Updating Subscription Management repositories. 2023-03-27T10:04:00.765 INFO:teuthology.orchestra.run.smithi159.stdout:Unable to read consumer identity 2023-03-27T10:04:01.015 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:00:02 ago on Mon 27 Mar 2023 10:03:58 AM UTC. 2023-03-27T10:04:01.054 INFO:teuthology.orchestra.run.smithi159.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-03-27T10:04:01.110 INFO:teuthology.orchestra.run.smithi159.stderr:Error: No packages marked for reinstall. 2023-03-27T10:04:01.155 DEBUG:teuthology.orchestra.run.smithi159:> rpm -q kernel --last | head -n 1 2023-03-27T10:04:01.236 INFO:teuthology.orchestra.run.smithi159.stdout:kernel-4.18.0-372.9.1.el8.x86_64 Thu 14 Jul 2022 06:23:21 PM UTC 2023-03-27T10:04:01.238 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-03-27T10:04:01.239 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-03-27T10:04:01.239 DEBUG:teuthology.parallel:result is None 2023-03-27T10:04:01.239 INFO:teuthology.run_tasks:Running task internal.base... 2023-03-27T10:04:01.251 INFO:teuthology.task.internal:Creating test directory... 2023-03-27T10:04:01.251 DEBUG:teuthology.orchestra.run.smithi159:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-03-27T10:04:01.254 DEBUG:teuthology.orchestra.run.smithi161:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-03-27T10:04:01.256 DEBUG:teuthology.orchestra.run.smithi171:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-03-27T10:04:01.278 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-03-27T10:04:01.301 INFO:teuthology.run_tasks:Running task internal.archive... 2023-03-27T10:04:01.346 INFO:teuthology.task.internal:Creating archive directory... 2023-03-27T10:04:01.346 DEBUG:teuthology.orchestra.run.smithi159:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-03-27T10:04:01.349 DEBUG:teuthology.orchestra.run.smithi161:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-03-27T10:04:01.351 DEBUG:teuthology.orchestra.run.smithi171:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-03-27T10:04:01.395 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-03-27T10:04:01.404 INFO:teuthology.task.internal:Enabling coredump saving... 2023-03-27T10:04:01.404 DEBUG:teuthology.orchestra.run.smithi161:> test -f /run/.containerenv -o -f /.dockerenv 2023-03-27T10:04:01.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-03-27T10:04:01.441 DEBUG:teuthology.orchestra.run.smithi171:> test -f /run/.containerenv -o -f /.dockerenv 2023-03-27T10:04:01.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-03-27T10:04:01.461 DEBUG:teuthology.orchestra.run.smithi159:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-03-27T10:04:01.463 DEBUG:teuthology.orchestra.run.smithi161:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-03-27T10:04:01.485 DEBUG:teuthology.orchestra.run.smithi171:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-03-27T10:04:01.504 INFO:teuthology.orchestra.run.smithi159.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-03-27T10:04:01.516 INFO:teuthology.orchestra.run.smithi159.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-03-27T10:04:01.529 INFO:teuthology.orchestra.run.smithi161.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-03-27T10:04:01.544 INFO:teuthology.orchestra.run.smithi161.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-03-27T10:04:01.554 INFO:teuthology.orchestra.run.smithi171.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-03-27T10:04:01.571 INFO:teuthology.orchestra.run.smithi171.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-03-27T10:04:01.573 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-03-27T10:04:01.581 INFO:teuthology.task.internal:Configuring sudo... 2023-03-27T10:04:01.581 DEBUG:teuthology.orchestra.run.smithi159:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-03-27T10:04:01.583 DEBUG:teuthology.orchestra.run.smithi161:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-03-27T10:04:01.585 DEBUG:teuthology.orchestra.run.smithi171:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-03-27T10:04:01.616 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-03-27T10:04:01.632 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-03-27T10:04:01.632 DEBUG:teuthology.orchestra.run.smithi159:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-03-27T10:04:01.649 DEBUG:teuthology.orchestra.run.smithi161:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-03-27T10:04:01.657 DEBUG:teuthology.orchestra.run.smithi171:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-03-27T10:04:01.675 DEBUG:teuthology.orchestra.run.smithi159:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-03-27T10:04:01.752 DEBUG:teuthology.orchestra.run.smithi159:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-03-27T10:04:01.792 DEBUG:teuthology.orchestra.run.smithi159:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-03-27T10:04:01.875 DEBUG:teuthology.orchestra.run.smithi159:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-03-27T10:04:01.911 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2023-03-27T10:04:01.911 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-03-27T10:04:01.979 DEBUG:teuthology.orchestra.run.smithi161:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-03-27T10:04:02.030 DEBUG:teuthology.orchestra.run.smithi161:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-03-27T10:04:02.089 DEBUG:teuthology.orchestra.run.smithi161:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-03-27T10:04:02.137 DEBUG:teuthology.orchestra.run.smithi161:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-03-27T10:04:02.178 DEBUG:teuthology.orchestra.run.smithi161:> set -ex 2023-03-27T10:04:02.178 DEBUG:teuthology.orchestra.run.smithi161:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-03-27T10:04:02.248 DEBUG:teuthology.orchestra.run.smithi171:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-03-27T10:04:02.299 DEBUG:teuthology.orchestra.run.smithi171:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-03-27T10:04:02.348 DEBUG:teuthology.orchestra.run.smithi171:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-03-27T10:04:02.438 DEBUG:teuthology.orchestra.run.smithi171:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-03-27T10:04:02.477 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2023-03-27T10:04:02.478 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-03-27T10:04:02.547 DEBUG:teuthology.orchestra.run.smithi159:> sudo service rsyslog restart 2023-03-27T10:04:02.550 DEBUG:teuthology.orchestra.run.smithi161:> sudo service rsyslog restart 2023-03-27T10:04:02.552 DEBUG:teuthology.orchestra.run.smithi171:> sudo service rsyslog restart 2023-03-27T10:04:02.603 INFO:teuthology.orchestra.run.smithi159.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-03-27T10:04:02.614 INFO:teuthology.orchestra.run.smithi161.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-03-27T10:04:02.686 INFO:teuthology.orchestra.run.smithi171.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-03-27T10:04:03.126 INFO:teuthology.run_tasks:Running task internal.timer... 2023-03-27T10:04:03.137 INFO:teuthology.task.internal:Starting timer... 2023-03-27T10:04:03.138 INFO:teuthology.run_tasks:Running task pcp... 2023-03-27T10:04:03.205 INFO:teuthology.run_tasks:Running task selinux... 2023-03-27T10:04:03.226 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:logrotate_t:s0']} 2023-03-27T10:04:03.227 DEBUG:teuthology.orchestra.run.smithi159:> sudo service auditd rotate 2023-03-27T10:04:03.285 INFO:teuthology.orchestra.run.smithi159.stdout:Rotating logs: 2023-03-27T10:04:03.286 DEBUG:teuthology.orchestra.run.smithi161:> sudo service auditd rotate 2023-03-27T10:04:03.345 INFO:teuthology.orchestra.run.smithi161.stdout:Rotating logs: 2023-03-27T10:04:03.348 DEBUG:teuthology.orchestra.run.smithi171:> sudo service auditd rotate 2023-03-27T10:04:03.415 INFO:teuthology.orchestra.run.smithi171.stdout:Rotating logs: 2023-03-27T10:04:03.416 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-03-27T10:04:03.417 DEBUG:teuthology.orchestra.run.smithi159:> /usr/sbin/getenforce 2023-03-27T10:04:03.448 INFO:teuthology.orchestra.run.smithi159.stdout:Permissive 2023-03-27T10:04:03.449 DEBUG:teuthology.orchestra.run.smithi161:> /usr/sbin/getenforce 2023-03-27T10:04:03.479 INFO:teuthology.orchestra.run.smithi161.stdout:Permissive 2023-03-27T10:04:03.480 DEBUG:teuthology.orchestra.run.smithi171:> /usr/sbin/getenforce 2023-03-27T10:04:03.505 INFO:teuthology.orchestra.run.smithi171.stdout:Permissive 2023-03-27T10:04:03.506 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi159.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi161.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi171.front.sepia.ceph.com': 'permissive'} 2023-03-27T10:04:03.506 DEBUG:teuthology.orchestra.run.smithi159:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-03-27T10:04:03.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-03-27T10:04:03.535 DEBUG:teuthology.orchestra.run.smithi161:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-03-27T10:04:03.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-03-27T10:04:03.562 DEBUG:teuthology.orchestra.run.smithi171:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-03-27T10:04:03.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-03-27T10:04:03.591 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-03-27T10:04:03.591 DEBUG:teuthology.orchestra.run.smithi159:> sudo /usr/sbin/setenforce permissive 2023-03-27T10:04:03.618 DEBUG:teuthology.orchestra.run.smithi161:> sudo /usr/sbin/setenforce permissive 2023-03-27T10:04:03.645 DEBUG:teuthology.orchestra.run.smithi171:> sudo /usr/sbin/setenforce permissive 2023-03-27T10:04:03.704 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-03-27T10:04:03.716 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main was just updated or references a specific commit; assuming it is current 2023-03-27T10:04:03.717 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-03-27T10:04:03.738 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-03-27T10:04:03.740 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi159.front.sepia.ceph.com,smithi161.front.sepia.ceph.com,smithi171.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-03-27T10:11:52.899 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi159.front.sepia.ceph.com'), Remote(name='ubuntu@smithi161.front.sepia.ceph.com'), Remote(name='ubuntu@smithi171.front.sepia.ceph.com')] 2023-03-27T10:11:52.901 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com' 2023-03-27T10:11:52.902 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-03-27T10:11:52.970 DEBUG:teuthology.orchestra.run.smithi159:> true 2023-03-27T10:11:52.994 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi161.front.sepia.ceph.com' 2023-03-27T10:11:52.995 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi161.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-03-27T10:11:53.064 DEBUG:teuthology.orchestra.run.smithi161:> true 2023-03-27T10:11:53.132 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi171.front.sepia.ceph.com' 2023-03-27T10:11:53.133 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi171.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-03-27T10:11:53.202 DEBUG:teuthology.orchestra.run.smithi171:> true 2023-03-27T10:11:53.272 INFO:teuthology.run_tasks:Running task clock... 2023-03-27T10:11:53.282 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-03-27T10:11:53.283 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-03-27T10:11:53.283 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-03-27T10:11:53.286 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-03-27T10:11:53.286 DEBUG:teuthology.orchestra.run.smithi161:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-03-27T10:11:53.288 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-03-27T10:11:53.288 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-03-27T10:11:53.316 INFO:teuthology.orchestra.run.smithi159.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-03-27T10:11:53.320 INFO:teuthology.orchestra.run.smithi161.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-03-27T10:11:53.331 INFO:teuthology.orchestra.run.smithi159.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-03-27T10:11:53.335 INFO:teuthology.orchestra.run.smithi161.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-03-27T10:11:53.346 INFO:teuthology.orchestra.run.smithi171.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-03-27T10:11:53.361 INFO:teuthology.orchestra.run.smithi171.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-03-27T10:11:53.410 INFO:teuthology.orchestra.run.smithi161.stderr:sudo: ntpd: command not found 2023-03-27T10:11:53.419 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: ntpd: command not found 2023-03-27T10:11:53.422 INFO:teuthology.orchestra.run.smithi161.stdout:506 Cannot talk to daemon 2023-03-27T10:11:53.429 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: ntpd: command not found 2023-03-27T10:11:53.431 INFO:teuthology.orchestra.run.smithi159.stdout:506 Cannot talk to daemon 2023-03-27T10:11:53.437 INFO:teuthology.orchestra.run.smithi161.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-03-27T10:11:53.442 INFO:teuthology.orchestra.run.smithi171.stdout:506 Cannot talk to daemon 2023-03-27T10:11:53.446 INFO:teuthology.orchestra.run.smithi159.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-03-27T10:11:53.451 INFO:teuthology.orchestra.run.smithi161.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-03-27T10:11:53.456 INFO:teuthology.orchestra.run.smithi171.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-03-27T10:11:53.460 INFO:teuthology.orchestra.run.smithi159.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-03-27T10:11:53.470 INFO:teuthology.orchestra.run.smithi171.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-03-27T10:11:53.527 INFO:teuthology.orchestra.run.smithi161.stderr:bash: ntpq: command not found 2023-03-27T10:11:53.530 INFO:teuthology.orchestra.run.smithi161.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-03-27T10:11:53.530 INFO:teuthology.orchestra.run.smithi161.stdout:=============================================================================== 2023-03-27T10:11:53.531 INFO:teuthology.orchestra.run.smithi161.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-03-27T10:11:53.531 INFO:teuthology.orchestra.run.smithi161.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-03-27T10:11:53.531 INFO:teuthology.orchestra.run.smithi161.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-03-27T10:11:53.531 INFO:teuthology.orchestra.run.smithi161.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-03-27T10:11:53.537 INFO:teuthology.orchestra.run.smithi159.stderr:bash: ntpq: command not found 2023-03-27T10:11:53.541 INFO:teuthology.orchestra.run.smithi159.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-03-27T10:11:53.541 INFO:teuthology.orchestra.run.smithi159.stdout:=============================================================================== 2023-03-27T10:11:53.541 INFO:teuthology.orchestra.run.smithi159.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-03-27T10:11:53.541 INFO:teuthology.orchestra.run.smithi159.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-03-27T10:11:53.541 INFO:teuthology.orchestra.run.smithi159.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-03-27T10:11:53.541 INFO:teuthology.orchestra.run.smithi159.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-03-27T10:11:53.542 INFO:teuthology.orchestra.run.smithi171.stderr:bash: ntpq: command not found 2023-03-27T10:11:53.544 INFO:teuthology.orchestra.run.smithi171.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-03-27T10:11:53.544 INFO:teuthology.orchestra.run.smithi171.stdout:=============================================================================== 2023-03-27T10:11:53.545 INFO:teuthology.orchestra.run.smithi171.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-03-27T10:11:53.545 INFO:teuthology.orchestra.run.smithi171.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-03-27T10:11:53.545 INFO:teuthology.orchestra.run.smithi171.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-03-27T10:11:53.545 INFO:teuthology.orchestra.run.smithi171.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-03-27T10:11:53.546 INFO:teuthology.run_tasks:Running task pexec... 2023-03-27T10:11:53.556 INFO:teuthology.task.pexec:Executing custom commands... 2023-03-27T10:11:53.557 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi159.front.sepia.ceph.com 2023-03-27T10:11:53.557 DEBUG:teuthology.orchestra.run.smithi159:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-03-27T10:11:53.558 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi161.front.sepia.ceph.com 2023-03-27T10:11:53.558 DEBUG:teuthology.orchestra.run.smithi161:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-03-27T10:11:53.558 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi171.front.sepia.ceph.com 2023-03-27T10:11:53.558 DEBUG:teuthology.orchestra.run.smithi171:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-03-27T10:11:53.943 INFO:teuthology.orchestra.run.smithi161.stdout:Updating Subscription Management repositories. 2023-03-27T10:11:53.954 INFO:teuthology.orchestra.run.smithi159.stdout:Updating Subscription Management repositories. 2023-03-27T10:11:53.956 INFO:teuthology.orchestra.run.smithi171.stdout:Updating Subscription Management repositories. 2023-03-27T10:11:54.760 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:39 ago on Mon 27 Mar 2023 10:10:15 AM UTC. 2023-03-27T10:11:54.780 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:44 ago on Mon 27 Mar 2023 10:10:10 AM UTC. 2023-03-27T10:11:54.792 INFO:teuthology.orchestra.run.smithi161.stdout:Last metadata expiration check: 0:01:26 ago on Mon 27 Mar 2023 10:10:28 AM UTC. 2023-03-27T10:11:56.495 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2023-03-27T10:11:56.496 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2023-03-27T10:11:56.496 INFO:teuthology.orchestra.run.smithi159.stdout: Package Architecture Version Repository Size 2023-03-27T10:11:56.497 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2023-03-27T10:11:56.497 INFO:teuthology.orchestra.run.smithi159.stdout:Resetting modules: 2023-03-27T10:11:56.497 INFO:teuthology.orchestra.run.smithi159.stdout: container-tools 2023-03-27T10:11:56.497 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:11:56.497 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2023-03-27T10:11:56.497 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2023-03-27T10:11:56.498 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:11:56.499 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2023-03-27T10:11:56.500 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2023-03-27T10:11:56.500 INFO:teuthology.orchestra.run.smithi171.stdout: Package Architecture Version Repository Size 2023-03-27T10:11:56.500 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2023-03-27T10:11:56.500 INFO:teuthology.orchestra.run.smithi171.stdout:Resetting modules: 2023-03-27T10:11:56.501 INFO:teuthology.orchestra.run.smithi171.stdout: container-tools 2023-03-27T10:11:56.501 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:11:56.501 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2023-03-27T10:11:56.501 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2023-03-27T10:11:56.501 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:11:56.506 INFO:teuthology.orchestra.run.smithi161.stdout:Dependencies resolved. 2023-03-27T10:11:56.507 INFO:teuthology.orchestra.run.smithi161.stdout:================================================================================ 2023-03-27T10:11:56.507 INFO:teuthology.orchestra.run.smithi161.stdout: Package Architecture Version Repository Size 2023-03-27T10:11:56.508 INFO:teuthology.orchestra.run.smithi161.stdout:================================================================================ 2023-03-27T10:11:56.508 INFO:teuthology.orchestra.run.smithi161.stdout:Resetting modules: 2023-03-27T10:11:56.508 INFO:teuthology.orchestra.run.smithi161.stdout: container-tools 2023-03-27T10:11:56.508 INFO:teuthology.orchestra.run.smithi161.stdout: 2023-03-27T10:11:56.508 INFO:teuthology.orchestra.run.smithi161.stdout:Transaction Summary 2023-03-27T10:11:56.508 INFO:teuthology.orchestra.run.smithi161.stdout:================================================================================ 2023-03-27T10:11:56.509 INFO:teuthology.orchestra.run.smithi161.stdout: 2023-03-27T10:11:57.452 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:47 ago on Mon 27 Mar 2023 10:10:10 AM UTC. 2023-03-27T10:11:57.462 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:42 ago on Mon 27 Mar 2023 10:10:15 AM UTC. 2023-03-27T10:11:57.486 INFO:teuthology.orchestra.run.smithi161.stdout:Last metadata expiration check: 0:01:29 ago on Mon 27 Mar 2023 10:10:28 AM UTC. 2023-03-27T10:12:00.134 INFO:teuthology.orchestra.run.smithi161.stdout:Complete! 2023-03-27T10:12:00.209 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2023-03-27T10:12:00.259 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2023-03-27T10:12:00.613 INFO:teuthology.orchestra.run.smithi161.stdout:Updating Subscription Management repositories. 2023-03-27T10:12:00.686 INFO:teuthology.orchestra.run.smithi159.stdout:Updating Subscription Management repositories. 2023-03-27T10:12:00.730 INFO:teuthology.orchestra.run.smithi171.stdout:Updating Subscription Management repositories. 2023-03-27T10:12:01.415 INFO:teuthology.orchestra.run.smithi161.stdout:Last metadata expiration check: 0:01:32 ago on Mon 27 Mar 2023 10:10:28 AM UTC. 2023-03-27T10:12:01.480 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:51 ago on Mon 27 Mar 2023 10:10:10 AM UTC. 2023-03-27T10:12:01.506 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:46 ago on Mon 27 Mar 2023 10:10:15 AM UTC. 2023-03-27T10:12:03.140 INFO:teuthology.orchestra.run.smithi161.stdout:Dependencies resolved. 2023-03-27T10:12:03.232 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2023-03-27T10:12:03.233 INFO:teuthology.orchestra.run.smithi161.stdout:============================================================================================================ 2023-03-27T10:12:03.233 INFO:teuthology.orchestra.run.smithi161.stdout: Package Arch Version Repository Size 2023-03-27T10:12:03.233 INFO:teuthology.orchestra.run.smithi161.stdout:============================================================================================================ 2023-03-27T10:12:03.233 INFO:teuthology.orchestra.run.smithi161.stdout:Installing group/module packages: 2023-03-27T10:12:03.234 INFO:teuthology.orchestra.run.smithi161.stdout: buildah x86_64 1:1.26.2-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 8.0 M 2023-03-27T10:12:03.234 INFO:teuthology.orchestra.run.smithi161.stdout: cockpit-podman noarch 49.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 524 k 2023-03-27T10:12:03.234 INFO:teuthology.orchestra.run.smithi161.stdout: crun x86_64 1.5-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 213 k 2023-03-27T10:12:03.234 INFO:teuthology.orchestra.run.smithi161.stdout: python3-podman noarch 4.0.1-1.module+el8.6.0+16837+85b82211 rhel-8-for-x86_64-appstream-rpms 149 k 2023-03-27T10:12:03.234 INFO:teuthology.orchestra.run.smithi161.stdout: skopeo x86_64 2:1.9.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 7.0 M 2023-03-27T10:12:03.234 INFO:teuthology.orchestra.run.smithi161.stdout: toolbox x86_64 0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-03-27T10:12:03.234 INFO:teuthology.orchestra.run.smithi161.stdout: udica noarch 0.2.6-4.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 49 k 2023-03-27T10:12:03.234 INFO:teuthology.orchestra.run.smithi161.stdout:Installing dependencies: 2023-03-27T10:12:03.235 INFO:teuthology.orchestra.run.smithi161.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2023-03-27T10:12:03.235 INFO:teuthology.orchestra.run.smithi161.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2023-03-27T10:12:03.235 INFO:teuthology.orchestra.run.smithi161.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-03-27T10:12:03.235 INFO:teuthology.orchestra.run.smithi161.stdout:Installing module profiles: 2023-03-27T10:12:03.235 INFO:teuthology.orchestra.run.smithi161.stdout: container-tools/common 2023-03-27T10:12:03.235 INFO:teuthology.orchestra.run.smithi161.stdout: 2023-03-27T10:12:03.235 INFO:teuthology.orchestra.run.smithi161.stdout:Enabling module streams: 2023-03-27T10:12:03.236 INFO:teuthology.orchestra.run.smithi161.stdout: container-tools rhel8 2023-03-27T10:12:03.236 INFO:teuthology.orchestra.run.smithi161.stdout: 2023-03-27T10:12:03.236 INFO:teuthology.orchestra.run.smithi161.stdout:Transaction Summary 2023-03-27T10:12:03.236 INFO:teuthology.orchestra.run.smithi161.stdout:============================================================================================================ 2023-03-27T10:12:03.236 INFO:teuthology.orchestra.run.smithi161.stdout:Install 10 Packages 2023-03-27T10:12:03.236 INFO:teuthology.orchestra.run.smithi161.stdout: 2023-03-27T10:12:03.237 INFO:teuthology.orchestra.run.smithi161.stdout:Total download size: 18 M 2023-03-27T10:12:03.237 INFO:teuthology.orchestra.run.smithi161.stdout:Installed size: 65 M 2023-03-27T10:12:03.237 INFO:teuthology.orchestra.run.smithi161.stdout:Downloading Packages: 2023-03-27T10:12:03.314 INFO:teuthology.orchestra.run.smithi161.stdout:(1/10): yajl-2.1.0-10.el8.x86_64.rpm 523 kB/s | 41 kB 00:00 2023-03-27T10:12:03.328 INFO:teuthology.orchestra.run.smithi159.stdout:============================================================================================================ 2023-03-27T10:12:03.328 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repository Size 2023-03-27T10:12:03.328 INFO:teuthology.orchestra.run.smithi159.stdout:============================================================================================================ 2023-03-27T10:12:03.328 INFO:teuthology.orchestra.run.smithi159.stdout:Installing group/module packages: 2023-03-27T10:12:03.328 INFO:teuthology.orchestra.run.smithi159.stdout: buildah x86_64 1:1.26.2-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 8.0 M 2023-03-27T10:12:03.328 INFO:teuthology.orchestra.run.smithi159.stdout: cockpit-podman noarch 49.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 524 k 2023-03-27T10:12:03.329 INFO:teuthology.orchestra.run.smithi159.stdout: crun x86_64 1.5-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 213 k 2023-03-27T10:12:03.329 INFO:teuthology.orchestra.run.smithi159.stdout: python3-podman noarch 4.0.1-1.module+el8.6.0+16837+85b82211 rhel-8-for-x86_64-appstream-rpms 149 k 2023-03-27T10:12:03.329 INFO:teuthology.orchestra.run.smithi159.stdout: skopeo x86_64 2:1.9.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 7.0 M 2023-03-27T10:12:03.329 INFO:teuthology.orchestra.run.smithi159.stdout: toolbox x86_64 0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-03-27T10:12:03.329 INFO:teuthology.orchestra.run.smithi159.stdout: udica noarch 0.2.6-4.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 49 k 2023-03-27T10:12:03.329 INFO:teuthology.orchestra.run.smithi159.stdout:Installing dependencies: 2023-03-27T10:12:03.329 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2023-03-27T10:12:03.330 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2023-03-27T10:12:03.330 INFO:teuthology.orchestra.run.smithi159.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-03-27T10:12:03.330 INFO:teuthology.orchestra.run.smithi159.stdout:Installing module profiles: 2023-03-27T10:12:03.330 INFO:teuthology.orchestra.run.smithi159.stdout: container-tools/common 2023-03-27T10:12:03.330 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:12:03.330 INFO:teuthology.orchestra.run.smithi159.stdout:Enabling module streams: 2023-03-27T10:12:03.330 INFO:teuthology.orchestra.run.smithi159.stdout: container-tools rhel8 2023-03-27T10:12:03.331 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:12:03.331 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2023-03-27T10:12:03.331 INFO:teuthology.orchestra.run.smithi159.stdout:============================================================================================================ 2023-03-27T10:12:03.331 INFO:teuthology.orchestra.run.smithi159.stdout:Install 10 Packages 2023-03-27T10:12:03.331 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:12:03.331 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 18 M 2023-03-27T10:12:03.331 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 65 M 2023-03-27T10:12:03.331 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2023-03-27T10:12:03.341 INFO:teuthology.orchestra.run.smithi161.stdout:(2/10): python3-pytoml-0.1.14-5.git7dea353.el8. 237 kB/s | 25 kB 00:00 2023-03-27T10:12:03.362 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2023-03-27T10:12:03.366 INFO:teuthology.orchestra.run.smithi161.stdout:(3/10): udica-0.2.6-4.module+el8.6.0+16771+28df 949 kB/s | 49 kB 00:00 2023-03-27T10:12:03.383 INFO:teuthology.orchestra.run.smithi161.stdout:(4/10): python3-pyxdg-0.25-16.el8.noarch.rpm 638 kB/s | 94 kB 00:00 2023-03-27T10:12:03.394 INFO:teuthology.orchestra.run.smithi159.stdout:(1/10): python3-pyxdg-0.25-16.el8.noarch.rpm 1.4 MB/s | 94 kB 00:00 2023-03-27T10:12:03.411 INFO:teuthology.orchestra.run.smithi159.stdout:(2/10): python3-pytoml-0.1.14-5.git7dea353.el8. 304 kB/s | 25 kB 00:00 2023-03-27T10:12:03.416 INFO:teuthology.orchestra.run.smithi161.stdout:(5/10): cockpit-podman-49.1-1.module+el8.6.0+16 6.8 MB/s | 524 kB 00:00 2023-03-27T10:12:03.428 INFO:teuthology.orchestra.run.smithi159.stdout:(3/10): yajl-2.1.0-10.el8.x86_64.rpm 413 kB/s | 41 kB 00:00 2023-03-27T10:12:03.433 INFO:teuthology.orchestra.run.smithi161.stdout:(6/10): python3-podman-4.0.1-1.module+el8.6.0+1 2.9 MB/s | 149 kB 00:00 2023-03-27T10:12:03.445 INFO:teuthology.orchestra.run.smithi159.stdout:(4/10): udica-0.2.6-4.module+el8.6.0+16771+28df 974 kB/s | 49 kB 00:00 2023-03-27T10:12:03.455 INFO:teuthology.orchestra.run.smithi171.stdout:============================================================================================================ 2023-03-27T10:12:03.455 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2023-03-27T10:12:03.455 INFO:teuthology.orchestra.run.smithi171.stdout:============================================================================================================ 2023-03-27T10:12:03.456 INFO:teuthology.orchestra.run.smithi171.stdout:Installing group/module packages: 2023-03-27T10:12:03.456 INFO:teuthology.orchestra.run.smithi171.stdout: buildah x86_64 1:1.26.2-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 8.0 M 2023-03-27T10:12:03.456 INFO:teuthology.orchestra.run.smithi171.stdout: cockpit-podman noarch 49.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 524 k 2023-03-27T10:12:03.456 INFO:teuthology.orchestra.run.smithi171.stdout: crun x86_64 1.5-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 213 k 2023-03-27T10:12:03.456 INFO:teuthology.orchestra.run.smithi171.stdout: python3-podman noarch 4.0.1-1.module+el8.6.0+16837+85b82211 rhel-8-for-x86_64-appstream-rpms 149 k 2023-03-27T10:12:03.456 INFO:teuthology.orchestra.run.smithi171.stdout: skopeo x86_64 2:1.9.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 7.0 M 2023-03-27T10:12:03.456 INFO:teuthology.orchestra.run.smithi171.stdout: toolbox x86_64 0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-03-27T10:12:03.457 INFO:teuthology.orchestra.run.smithi171.stdout: udica noarch 0.2.6-4.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 49 k 2023-03-27T10:12:03.457 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2023-03-27T10:12:03.457 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2023-03-27T10:12:03.457 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2023-03-27T10:12:03.457 INFO:teuthology.orchestra.run.smithi171.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-03-27T10:12:03.457 INFO:teuthology.orchestra.run.smithi171.stdout:Installing module profiles: 2023-03-27T10:12:03.457 INFO:teuthology.orchestra.run.smithi171.stdout: container-tools/common 2023-03-27T10:12:03.458 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:12:03.458 INFO:teuthology.orchestra.run.smithi171.stdout:Enabling module streams: 2023-03-27T10:12:03.458 INFO:teuthology.orchestra.run.smithi171.stdout: container-tools rhel8 2023-03-27T10:12:03.458 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:12:03.458 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2023-03-27T10:12:03.458 INFO:teuthology.orchestra.run.smithi171.stdout:============================================================================================================ 2023-03-27T10:12:03.458 INFO:teuthology.orchestra.run.smithi171.stdout:Install 10 Packages 2023-03-27T10:12:03.459 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:12:03.459 INFO:teuthology.orchestra.run.smithi161.stdout:(7/10): crun-1.5-1.module+el8.6.0+16771+28dfca7 5.0 MB/s | 213 kB 00:00 2023-03-27T10:12:03.459 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 18 M 2023-03-27T10:12:03.460 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 65 M 2023-03-27T10:12:03.460 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2023-03-27T10:12:03.479 INFO:teuthology.orchestra.run.smithi159.stdout:(5/10): cockpit-podman-49.1-1.module+el8.6.0+16 7.6 MB/s | 524 kB 00:00 2023-03-27T10:12:03.504 INFO:teuthology.orchestra.run.smithi159.stdout:(6/10): python3-podman-4.0.1-1.module+el8.6.0+1 2.5 MB/s | 149 kB 00:00 2023-03-27T10:12:03.522 INFO:teuthology.orchestra.run.smithi171.stdout:(1/10): python3-pytoml-0.1.14-5.git7dea353.el8. 386 kB/s | 25 kB 00:00 2023-03-27T10:12:03.538 INFO:teuthology.orchestra.run.smithi159.stdout:(7/10): crun-1.5-1.module+el8.6.0+16771+28dfca7 3.6 MB/s | 213 kB 00:00 2023-03-27T10:12:03.548 INFO:teuthology.orchestra.run.smithi171.stdout:(2/10): yajl-2.1.0-10.el8.x86_64.rpm 455 kB/s | 41 kB 00:00 2023-03-27T10:12:03.551 INFO:teuthology.orchestra.run.smithi161.stdout:(8/10): toolbox-0.0.99.3-0.6.module+el8.6.0+167 24 MB/s | 2.2 MB 00:00 2023-03-27T10:12:03.573 INFO:teuthology.orchestra.run.smithi171.stdout:(3/10): python3-pyxdg-0.25-16.el8.noarch.rpm 817 kB/s | 94 kB 00:00 2023-03-27T10:12:03.598 INFO:teuthology.orchestra.run.smithi171.stdout:(4/10): udica-0.2.6-4.module+el8.6.0+16771+28df 654 kB/s | 49 kB 00:00 2023-03-27T10:12:03.640 INFO:teuthology.orchestra.run.smithi171.stdout:(5/10): cockpit-podman-49.1-1.module+el8.6.0+16 5.5 MB/s | 524 kB 00:00 2023-03-27T10:12:03.646 INFO:teuthology.orchestra.run.smithi159.stdout:(8/10): toolbox-0.0.99.3-0.6.module+el8.6.0+167 20 MB/s | 2.2 MB 00:00 2023-03-27T10:12:03.666 INFO:teuthology.orchestra.run.smithi171.stdout:(6/10): python3-podman-4.0.1-1.module+el8.6.0+1 2.2 MB/s | 149 kB 00:00 2023-03-27T10:12:03.668 INFO:teuthology.orchestra.run.smithi161.stdout:(9/10): skopeo-1.9.1-1.module+el8.6.0+16771+28d 30 MB/s | 7.0 MB 00:00 2023-03-27T10:12:03.691 INFO:teuthology.orchestra.run.smithi171.stdout:(7/10): crun-1.5-1.module+el8.6.0+16771+28dfca7 4.2 MB/s | 213 kB 00:00 2023-03-27T10:12:03.759 INFO:teuthology.orchestra.run.smithi161.stdout:(10/10): buildah-1.26.2-1.module+el8.6.0+16771+ 20 MB/s | 8.0 MB 00:00 2023-03-27T10:12:03.760 INFO:teuthology.orchestra.run.smithi161.stdout:-------------------------------------------------------------------------------- 2023-03-27T10:12:03.760 INFO:teuthology.orchestra.run.smithi161.stdout:Total 35 MB/s | 18 MB 00:00 2023-03-27T10:12:03.763 INFO:teuthology.orchestra.run.smithi159.stdout:(9/10): skopeo-1.9.1-1.module+el8.6.0+16771+28d 27 MB/s | 7.0 MB 00:00 2023-03-27T10:12:03.774 INFO:teuthology.orchestra.run.smithi171.stdout:(8/10): toolbox-0.0.99.3-0.6.module+el8.6.0+167 26 MB/s | 2.2 MB 00:00 2023-03-27T10:12:03.889 INFO:teuthology.orchestra.run.smithi159.stdout:(10/10): buildah-1.26.2-1.module+el8.6.0+16771+ 17 MB/s | 8.0 MB 00:00 2023-03-27T10:12:03.889 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2023-03-27T10:12:03.890 INFO:teuthology.orchestra.run.smithi159.stdout:Total 33 MB/s | 18 MB 00:00 2023-03-27T10:12:03.891 INFO:teuthology.orchestra.run.smithi171.stdout:(9/10): skopeo-1.9.1-1.module+el8.6.0+16771+28d 31 MB/s | 7.0 MB 00:00 2023-03-27T10:12:03.962 INFO:teuthology.orchestra.run.smithi161.stdout:Running transaction check 2023-03-27T10:12:03.981 INFO:teuthology.orchestra.run.smithi161.stdout:Transaction check succeeded. 2023-03-27T10:12:03.981 INFO:teuthology.orchestra.run.smithi161.stdout:Running transaction test 2023-03-27T10:12:04.018 INFO:teuthology.orchestra.run.smithi171.stdout:(10/10): buildah-1.26.2-1.module+el8.6.0+16771+ 18 MB/s | 8.0 MB 00:00 2023-03-27T10:12:04.019 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2023-03-27T10:12:04.019 INFO:teuthology.orchestra.run.smithi171.stdout:Total 32 MB/s | 18 MB 00:00 2023-03-27T10:12:04.092 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2023-03-27T10:12:04.111 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2023-03-27T10:12:04.111 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2023-03-27T10:12:04.165 INFO:teuthology.orchestra.run.smithi161.stdout:Transaction test succeeded. 2023-03-27T10:12:04.170 INFO:teuthology.orchestra.run.smithi161.stdout:Running transaction 2023-03-27T10:12:04.220 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2023-03-27T10:12:04.239 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2023-03-27T10:12:04.239 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2023-03-27T10:12:04.293 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2023-03-27T10:12:04.297 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2023-03-27T10:12:04.420 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2023-03-27T10:12:04.424 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2023-03-27T10:12:04.719 INFO:teuthology.orchestra.run.smithi161.stdout: Preparing : 1/1 2023-03-27T10:12:04.860 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2023-03-27T10:12:04.894 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/10 2023-03-27T10:12:04.951 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2023-03-27T10:12:05.068 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/10 2023-03-27T10:12:05.094 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : yajl-2.1.0-10.el8.x86_64 2/10 2023-03-27T10:12:05.178 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/10 2023-03-27T10:12:05.260 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : yajl-2.1.0-10.el8.x86_64 2/10 2023-03-27T10:12:05.280 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/10 2023-03-27T10:12:05.356 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : yajl-2.1.0-10.el8.x86_64 2/10 2023-03-27T10:12:05.471 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/10 2023-03-27T10:12:05.491 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 4/10 2023-03-27T10:12:05.559 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/10 2023-03-27T10:12:05.723 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 4/10 2023-03-27T10:12:05.783 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 4/10 2023-03-27T10:12:05.865 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 5/10 2023-03-27T10:12:06.122 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 5/10 2023-03-27T10:12:06.137 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 5/10 2023-03-27T10:12:06.629 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 6/10 2023-03-27T10:12:06.930 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 6/10 2023-03-27T10:12:06.942 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 6/10 2023-03-27T10:12:07.535 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 7/10 2023-03-27T10:12:07.753 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 8/10 2023-03-27T10:12:07.815 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 7/10 2023-03-27T10:12:07.893 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 7/10 2023-03-27T10:12:07.943 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 9/10 2023-03-27T10:12:08.037 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 8/10 2023-03-27T10:12:08.139 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 8/10 2023-03-27T10:12:08.170 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-03-27T10:12:08.205 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 9/10 2023-03-27T10:12:08.289 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-03-27T10:12:08.289 INFO:teuthology.orchestra.run.smithi161.stdout:[/usr/lib/tmpfiles.d/journal-nocow.conf:26] Failed to resolve specifier: uninitialized /etc detected, skipping 2023-03-27T10:12:08.289 INFO:teuthology.orchestra.run.smithi161.stdout:All rules containing unresolvable specifiers will be skipped. 2023-03-27T10:12:08.289 INFO:teuthology.orchestra.run.smithi161.stdout: 2023-03-27T10:12:08.350 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 9/10 2023-03-27T10:12:08.381 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-03-27T10:12:08.441 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 1/10 2023-03-27T10:12:08.441 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 2/10 2023-03-27T10:12:08.441 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/10 2023-03-27T10:12:08.441 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 4/10 2023-03-27T10:12:08.441 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 5/10 2023-03-27T10:12:08.442 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 6/10 2023-03-27T10:12:08.442 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 7/10 2023-03-27T10:12:08.442 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 8/10 2023-03-27T10:12:08.442 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 9/10 2023-03-27T10:12:08.487 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-03-27T10:12:08.487 INFO:teuthology.orchestra.run.smithi171.stdout:[/usr/lib/tmpfiles.d/journal-nocow.conf:26] Failed to resolve specifier: uninitialized /etc detected, skipping 2023-03-27T10:12:08.487 INFO:teuthology.orchestra.run.smithi171.stdout:All rules containing unresolvable specifiers will be skipped. 2023-03-27T10:12:08.487 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:12:08.577 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-03-27T10:12:08.646 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 1/10 2023-03-27T10:12:08.646 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 2/10 2023-03-27T10:12:08.646 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/10 2023-03-27T10:12:08.646 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 4/10 2023-03-27T10:12:08.646 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 5/10 2023-03-27T10:12:08.647 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 6/10 2023-03-27T10:12:08.647 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 7/10 2023-03-27T10:12:08.647 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 8/10 2023-03-27T10:12:08.647 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 9/10 2023-03-27T10:12:08.691 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-03-27T10:12:08.691 INFO:teuthology.orchestra.run.smithi159.stdout:[/usr/lib/tmpfiles.d/journal-nocow.conf:26] Failed to resolve specifier: uninitialized /etc detected, skipping 2023-03-27T10:12:08.691 INFO:teuthology.orchestra.run.smithi159.stdout:All rules containing unresolvable specifiers will be skipped. 2023-03-27T10:12:08.692 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:12:08.846 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 1/10 2023-03-27T10:12:08.846 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 2/10 2023-03-27T10:12:08.846 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/10 2023-03-27T10:12:08.846 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 4/10 2023-03-27T10:12:08.846 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 5/10 2023-03-27T10:12:08.846 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 6/10 2023-03-27T10:12:08.847 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 7/10 2023-03-27T10:12:08.847 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 8/10 2023-03-27T10:12:08.847 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 9/10 2023-03-27T10:12:09.590 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 10/10 2023-03-27T10:12:09.591 INFO:teuthology.orchestra.run.smithi161.stdout:Last metadata expiration check: 0:01:41 ago on Mon 27 Mar 2023 10:10:28 AM UTC. 2023-03-27T10:12:09.773 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 10/10 2023-03-27T10:12:09.773 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:54 ago on Mon 27 Mar 2023 10:10:15 AM UTC. 2023-03-27T10:12:10.056 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 10/10 2023-03-27T10:12:10.056 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:59 ago on Mon 27 Mar 2023 10:10:10 AM UTC. 2023-03-27T10:12:12.645 INFO:teuthology.orchestra.run.smithi161.stdout:Installed products updated. 2023-03-27T10:12:12.844 INFO:teuthology.orchestra.run.smithi161.stdout: 2023-03-27T10:12:12.844 INFO:teuthology.orchestra.run.smithi161.stdout:Installed: 2023-03-27T10:12:12.844 INFO:teuthology.orchestra.run.smithi161.stdout: buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x86_64 2023-03-27T10:12:12.845 INFO:teuthology.orchestra.run.smithi161.stdout: cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca77.noarch 2023-03-27T10:12:12.845 INFO:teuthology.orchestra.run.smithi161.stdout: crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 2023-03-27T10:12:12.845 INFO:teuthology.orchestra.run.smithi161.stdout: python3-podman-4.0.1-1.module+el8.6.0+16837+85b82211.noarch 2023-03-27T10:12:12.845 INFO:teuthology.orchestra.run.smithi161.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-03-27T10:12:12.845 INFO:teuthology.orchestra.run.smithi161.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-03-27T10:12:12.845 INFO:teuthology.orchestra.run.smithi161.stdout: skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86_64 2023-03-27T10:12:12.845 INFO:teuthology.orchestra.run.smithi161.stdout: toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77.x86_64 2023-03-27T10:12:12.845 INFO:teuthology.orchestra.run.smithi161.stdout: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 2023-03-27T10:12:12.846 INFO:teuthology.orchestra.run.smithi161.stdout: yajl-2.1.0-10.el8.x86_64 2023-03-27T10:12:12.846 INFO:teuthology.orchestra.run.smithi161.stdout: 2023-03-27T10:12:12.846 INFO:teuthology.orchestra.run.smithi161.stdout:Complete! 2023-03-27T10:12:12.913 INFO:teuthology.orchestra.run.smithi171.stdout:Installed products updated. 2023-03-27T10:12:12.930 INFO:teuthology.orchestra.run.smithi159.stdout:Installed products updated. 2023-03-27T10:12:13.003 DEBUG:teuthology.parallel:result is None 2023-03-27T10:12:13.099 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:12:13.100 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2023-03-27T10:12:13.100 INFO:teuthology.orchestra.run.smithi171.stdout: buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x86_64 2023-03-27T10:12:13.100 INFO:teuthology.orchestra.run.smithi171.stdout: cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca77.noarch 2023-03-27T10:12:13.100 INFO:teuthology.orchestra.run.smithi171.stdout: crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 2023-03-27T10:12:13.100 INFO:teuthology.orchestra.run.smithi171.stdout: python3-podman-4.0.1-1.module+el8.6.0+16837+85b82211.noarch 2023-03-27T10:12:13.100 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-03-27T10:12:13.100 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-03-27T10:12:13.101 INFO:teuthology.orchestra.run.smithi171.stdout: skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86_64 2023-03-27T10:12:13.101 INFO:teuthology.orchestra.run.smithi171.stdout: toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77.x86_64 2023-03-27T10:12:13.101 INFO:teuthology.orchestra.run.smithi171.stdout: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 2023-03-27T10:12:13.101 INFO:teuthology.orchestra.run.smithi171.stdout: yajl-2.1.0-10.el8.x86_64 2023-03-27T10:12:13.101 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:12:13.101 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2023-03-27T10:12:13.135 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:12:13.135 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2023-03-27T10:12:13.135 INFO:teuthology.orchestra.run.smithi159.stdout: buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x86_64 2023-03-27T10:12:13.135 INFO:teuthology.orchestra.run.smithi159.stdout: cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca77.noarch 2023-03-27T10:12:13.136 INFO:teuthology.orchestra.run.smithi159.stdout: crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 2023-03-27T10:12:13.136 INFO:teuthology.orchestra.run.smithi159.stdout: python3-podman-4.0.1-1.module+el8.6.0+16837+85b82211.noarch 2023-03-27T10:12:13.136 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-03-27T10:12:13.136 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-03-27T10:12:13.136 INFO:teuthology.orchestra.run.smithi159.stdout: skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86_64 2023-03-27T10:12:13.136 INFO:teuthology.orchestra.run.smithi159.stdout: toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77.x86_64 2023-03-27T10:12:13.136 INFO:teuthology.orchestra.run.smithi159.stdout: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 2023-03-27T10:12:13.136 INFO:teuthology.orchestra.run.smithi159.stdout: yajl-2.1.0-10.el8.x86_64 2023-03-27T10:12:13.137 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:12:13.137 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2023-03-27T10:12:13.249 DEBUG:teuthology.parallel:result is None 2023-03-27T10:12:13.269 DEBUG:teuthology.parallel:result is None 2023-03-27T10:12:13.270 INFO:teuthology.run_tasks:Running task install... 2023-03-27T10:12:13.279 DEBUG:teuthology.task.install:project ceph 2023-03-27T10:12:13.279 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b70fdf973c2ffa23c0d8aa6918519da05d922302'}} 2023-03-27T10:12:13.280 DEBUG:teuthology.task.install:config {'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'flavor': 'default', 'sha1': 'b70fdf973c2ffa23c0d8aa6918519da05d922302'} 2023-03-27T10:12:13.280 INFO:teuthology.task.install:Using flavor: default 2023-03-27T10:12:13.286 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-03-27T10:12:13.286 INFO:teuthology.task.install:extra packages: {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-top', 'cephfs-mirror']} 2023-03-27T10:12:13.286 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b70fdf973c2ffa23c0d8aa6918519da05d922302', 'tag': None, 'wait_for_package': False} 2023-03-27T10:12:13.287 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=b70fdf973c2ffa23c0d8aa6918519da05d922302 2023-03-27T10:12:13.290 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b70fdf973c2ffa23c0d8aa6918519da05d922302', 'tag': None, 'wait_for_package': False} 2023-03-27T10:12:13.290 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=b70fdf973c2ffa23c0d8aa6918519da05d922302 2023-03-27T10:12:13.291 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b70fdf973c2ffa23c0d8aa6918519da05d922302', 'tag': None, 'wait_for_package': False} 2023-03-27T10:12:13.291 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=b70fdf973c2ffa23c0d8aa6918519da05d922302 2023-03-27T10:12:13.430 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/flavors/default/ 2023-03-27T10:12:13.431 INFO:teuthology.task.install.rpm:Package version is 18.0.0-2533.gb70fdf97 2023-03-27T10:12:13.440 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/flavors/default/ 2023-03-27T10:12:13.440 INFO:teuthology.task.install.rpm:Package version is 18.0.0-2533.gb70fdf97 2023-03-27T10:12:13.444 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/flavors/default/ 2023-03-27T10:12:13.444 INFO:teuthology.task.install.rpm:Package version is 18.0.0-2533.gb70fdf97 2023-03-27T10:12:13.588 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-03-27T10:12:13.588 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2023-03-27T10:12:13.589 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-03-27T10:12:13.591 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-03-27T10:12:13.591 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2023-03-27T10:12:13.592 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-03-27T10:12:13.601 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-03-27T10:12:13.601 DEBUG:teuthology.orchestra.run.smithi161:> set -ex 2023-03-27T10:12:13.601 DEBUG:teuthology.orchestra.run.smithi161:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-03-27T10:12:13.616 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2023-03-27T10:12:13.616 DEBUG:teuthology.orchestra.run.smithi159:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/b70fdf973c2ffa23c0d8aa6918519da05d922302/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-03-27T10:12:13.620 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2023-03-27T10:12:13.620 DEBUG:teuthology.orchestra.run.smithi171:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/b70fdf973c2ffa23c0d8aa6918519da05d922302/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-03-27T10:12:13.631 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2023-03-27T10:12:13.631 DEBUG:teuthology.orchestra.run.smithi161:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/b70fdf973c2ffa23c0d8aa6918519da05d922302/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-03-27T10:12:13.683 DEBUG:teuthology.orchestra.run.smithi159:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-03-27T10:12:13.688 DEBUG:teuthology.orchestra.run.smithi171:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-03-27T10:12:13.701 DEBUG:teuthology.orchestra.run.smithi161:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-03-27T10:12:13.760 DEBUG:teuthology.orchestra.run.smithi159:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-03-27T10:12:13.765 DEBUG:teuthology.orchestra.run.smithi171:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-03-27T10:12:13.780 DEBUG:teuthology.orchestra.run.smithi161:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-03-27T10:12:13.826 INFO:teuthology.orchestra.run.smithi159.stdout:check_obsoletes = 1 2023-03-27T10:12:13.827 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum clean all 2023-03-27T10:12:13.830 INFO:teuthology.orchestra.run.smithi171.stdout:check_obsoletes = 1 2023-03-27T10:12:13.832 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum clean all 2023-03-27T10:12:13.846 INFO:teuthology.orchestra.run.smithi161.stdout:check_obsoletes = 1 2023-03-27T10:12:13.848 DEBUG:teuthology.orchestra.run.smithi161:> sudo yum clean all 2023-03-27T10:12:14.201 INFO:teuthology.orchestra.run.smithi159.stdout:Updating Subscription Management repositories. 2023-03-27T10:12:14.204 INFO:teuthology.orchestra.run.smithi171.stdout:Updating Subscription Management repositories. 2023-03-27T10:12:14.222 INFO:teuthology.orchestra.run.smithi161.stdout:Updating Subscription Management repositories. 2023-03-27T10:12:14.452 INFO:teuthology.orchestra.run.smithi171.stdout:44 files removed 2023-03-27T10:12:14.460 INFO:teuthology.orchestra.run.smithi159.stdout:44 files removed 2023-03-27T10:12:14.469 INFO:teuthology.orchestra.run.smithi161.stdout:44 files removed 2023-03-27T10:12:14.500 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2023-03-27T10:12:14.507 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2023-03-27T10:12:14.516 DEBUG:teuthology.orchestra.run.smithi161:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2023-03-27T10:12:14.830 INFO:teuthology.orchestra.run.smithi171.stdout:Updating Subscription Management repositories. 2023-03-27T10:12:14.838 INFO:teuthology.orchestra.run.smithi159.stdout:Updating Subscription Management repositories. 2023-03-27T10:12:14.850 INFO:teuthology.orchestra.run.smithi161.stdout:Updating Subscription Management repositories. 2023-03-27T10:12:15.845 INFO:teuthology.orchestra.run.smithi171.stdout:ceph packages for x86_64 125 kB/s | 79 kB 00:00 2023-03-27T10:12:15.880 INFO:teuthology.orchestra.run.smithi161.stdout:ceph packages for x86_64 124 kB/s | 79 kB 00:00 2023-03-27T10:12:15.883 INFO:teuthology.orchestra.run.smithi159.stdout:ceph packages for x86_64 124 kB/s | 79 kB 00:00 2023-03-27T10:12:16.312 INFO:teuthology.orchestra.run.smithi171.stdout:ceph noarch packages 21 kB/s | 9.3 kB 00:00 2023-03-27T10:12:16.339 INFO:teuthology.orchestra.run.smithi161.stdout:ceph noarch packages 22 kB/s | 9.3 kB 00:00 2023-03-27T10:12:16.358 INFO:teuthology.orchestra.run.smithi159.stdout:ceph noarch packages 21 kB/s | 9.3 kB 00:00 2023-03-27T10:12:16.796 INFO:teuthology.orchestra.run.smithi171.stdout:ceph source packages 3.2 kB/s | 1.5 kB 00:00 2023-03-27T10:12:16.798 INFO:teuthology.orchestra.run.smithi161.stdout:ceph source packages 3.4 kB/s | 1.5 kB 00:00 2023-03-27T10:12:16.834 INFO:teuthology.orchestra.run.smithi159.stdout:ceph source packages 3.3 kB/s | 1.5 kB 00:00 2023-03-27T10:12:16.955 INFO:teuthology.orchestra.run.smithi171.stdout:Copr repo for python3-asyncssh owned by ceph 25 kB/s | 3.5 kB 00:00 2023-03-27T10:12:16.957 INFO:teuthology.orchestra.run.smithi161.stdout:Copr repo for python3-asyncssh owned by ceph 25 kB/s | 3.5 kB 00:00 2023-03-27T10:12:16.985 INFO:teuthology.orchestra.run.smithi159.stdout:Copr repo for python3-asyncssh owned by ceph 26 kB/s | 3.5 kB 00:00 2023-03-27T10:12:18.031 INFO:teuthology.orchestra.run.smithi171.stdout:Extra Packages for Enterprise Linux 13 MB/s | 14 MB 00:01 2023-03-27T10:12:18.117 INFO:teuthology.orchestra.run.smithi159.stdout:Extra Packages for Enterprise Linux 12 MB/s | 14 MB 00:01 2023-03-27T10:12:18.118 INFO:teuthology.orchestra.run.smithi161.stdout:Extra Packages for Enterprise Linux 12 MB/s | 14 MB 00:01 2023-03-27T10:12:21.139 INFO:teuthology.orchestra.run.smithi171.stdout:lab-extras 356 kB/s | 24 kB 00:00 2023-03-27T10:12:21.169 INFO:teuthology.orchestra.run.smithi159.stdout:lab-extras 481 kB/s | 24 kB 00:00 2023-03-27T10:12:21.241 INFO:teuthology.orchestra.run.smithi161.stdout:lab-extras 352 kB/s | 24 kB 00:00 2023-03-27T10:12:22.063 INFO:teuthology.orchestra.run.smithi159.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 57 MB/s | 47 MB 00:00 2023-03-27T10:12:22.107 INFO:teuthology.orchestra.run.smithi171.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 59 MB/s | 53 MB 00:00 2023-03-27T10:12:22.168 INFO:teuthology.orchestra.run.smithi161.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 62 MB/s | 53 MB 00:00 2023-03-27T10:12:30.014 INFO:teuthology.orchestra.run.smithi159.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 57 MB/s | 53 MB 00:00 2023-03-27T10:12:30.217 INFO:teuthology.orchestra.run.smithi171.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 56 MB/s | 47 MB 00:00 2023-03-27T10:12:30.277 INFO:teuthology.orchestra.run.smithi161.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 54 MB/s | 47 MB 00:00 2023-03-27T10:12:41.090 INFO:teuthology.orchestra.run.smithi171.stdout:Package librados2-1:12.2.7-9.el8.x86_64 is already installed. 2023-03-27T10:12:41.091 INFO:teuthology.orchestra.run.smithi171.stdout:Package librbd1-1:12.2.7-9.el8.x86_64 is already installed. 2023-03-27T10:12:41.093 INFO:teuthology.orchestra.run.smithi161.stdout:Package librados2-1:12.2.7-9.el8.x86_64 is already installed. 2023-03-27T10:12:41.094 INFO:teuthology.orchestra.run.smithi161.stdout:Package librbd1-1:12.2.7-9.el8.x86_64 is already installed. 2023-03-27T10:12:41.105 INFO:teuthology.orchestra.run.smithi171.stdout:Package NetworkManager-1:1.36.0-4.el8.x86_64 is already installed. 2023-03-27T10:12:41.106 INFO:teuthology.orchestra.run.smithi171.stdout:Package iproute-5.15.0-4.el8.x86_64 is already installed. 2023-03-27T10:12:41.107 INFO:teuthology.orchestra.run.smithi171.stdout:Package util-linux-2.32.1-35.el8.x86_64 is already installed. 2023-03-27T10:12:41.108 INFO:teuthology.orchestra.run.smithi161.stdout:Package NetworkManager-1:1.36.0-4.el8.x86_64 is already installed. 2023-03-27T10:12:41.108 INFO:teuthology.orchestra.run.smithi171.stdout:Package libacl-devel-2.2.53-1.el8.x86_64 is already installed. 2023-03-27T10:12:41.109 INFO:teuthology.orchestra.run.smithi161.stdout:Package iproute-5.15.0-4.el8.x86_64 is already installed. 2023-03-27T10:12:41.110 INFO:teuthology.orchestra.run.smithi161.stdout:Package util-linux-2.32.1-35.el8.x86_64 is already installed. 2023-03-27T10:12:41.111 INFO:teuthology.orchestra.run.smithi171.stdout:Package libattr-devel-2.4.48-3.el8.x86_64 is already installed. 2023-03-27T10:12:41.111 INFO:teuthology.orchestra.run.smithi161.stdout:Package libacl-devel-2.2.53-1.el8.x86_64 is already installed. 2023-03-27T10:12:41.113 INFO:teuthology.orchestra.run.smithi161.stdout:Package libattr-devel-2.4.48-3.el8.x86_64 is already installed. 2023-03-27T10:12:41.114 INFO:teuthology.orchestra.run.smithi171.stdout:Package xfsdump-3.1.8-4.el8.x86_64 is already installed. 2023-03-27T10:12:41.115 INFO:teuthology.orchestra.run.smithi171.stdout:Package xfsprogs-5.0.0-10.el8.x86_64 is already installed. 2023-03-27T10:12:41.116 INFO:teuthology.orchestra.run.smithi161.stdout:Package xfsdump-3.1.8-4.el8.x86_64 is already installed. 2023-03-27T10:12:41.117 INFO:teuthology.orchestra.run.smithi161.stdout:Package xfsprogs-5.0.0-10.el8.x86_64 is already installed. 2023-03-27T10:12:41.187 INFO:teuthology.orchestra.run.smithi159.stdout:Package librados2-1:12.2.7-9.el8.x86_64 is already installed. 2023-03-27T10:12:41.188 INFO:teuthology.orchestra.run.smithi159.stdout:Package librbd1-1:12.2.7-9.el8.x86_64 is already installed. 2023-03-27T10:12:41.203 INFO:teuthology.orchestra.run.smithi159.stdout:Package NetworkManager-1:1.36.0-4.el8.x86_64 is already installed. 2023-03-27T10:12:41.204 INFO:teuthology.orchestra.run.smithi159.stdout:Package iproute-5.15.0-4.el8.x86_64 is already installed. 2023-03-27T10:12:41.206 INFO:teuthology.orchestra.run.smithi159.stdout:Package util-linux-2.32.1-35.el8.x86_64 is already installed. 2023-03-27T10:12:41.207 INFO:teuthology.orchestra.run.smithi159.stdout:Package libacl-devel-2.2.53-1.el8.x86_64 is already installed. 2023-03-27T10:12:41.209 INFO:teuthology.orchestra.run.smithi159.stdout:Package libattr-devel-2.4.48-3.el8.x86_64 is already installed. 2023-03-27T10:12:41.213 INFO:teuthology.orchestra.run.smithi159.stdout:Package xfsdump-3.1.8-4.el8.x86_64 is already installed. 2023-03-27T10:12:41.214 INFO:teuthology.orchestra.run.smithi159.stdout:Package xfsprogs-5.0.0-10.el8.x86_64 is already installed. 2023-03-27T10:12:41.248 INFO:teuthology.orchestra.run.smithi161.stdout:Dependencies resolved. 2023-03-27T10:12:41.251 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2023-03-27T10:12:41.260 INFO:teuthology.orchestra.run.smithi161.stdout:=========================================================================================================================== 2023-03-27T10:12:41.260 INFO:teuthology.orchestra.run.smithi161.stdout: Package Arch Version Repository Size 2023-03-27T10:12:41.260 INFO:teuthology.orchestra.run.smithi161.stdout:=========================================================================================================================== 2023-03-27T10:12:41.260 INFO:teuthology.orchestra.run.smithi161.stdout:Installing: 2023-03-27T10:12:41.261 INFO:teuthology.orchestra.run.smithi161.stdout: NetworkManager-initscripts-updown noarch 1:1.36.0-9.el8_6 rhel-8-for-x86_64-baseos-rpms 138 k 2023-03-27T10:12:41.261 INFO:teuthology.orchestra.run.smithi161.stdout: bison x86_64 3.0.4-10.el8 rhel-8-for-x86_64-appstream-rpms 688 k 2023-03-27T10:12:41.261 INFO:teuthology.orchestra.run.smithi161.stdout: ceph x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 6.4 k 2023-03-27T10:12:41.261 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-base x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 4.9 M 2023-03-27T10:12:41.262 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-fuse x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 845 k 2023-03-27T10:12:41.262 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-immutable-object-cache x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 165 k 2023-03-27T10:12:41.262 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-mgr x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 1.5 M 2023-03-27T10:12:41.262 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-mgr-cephadm noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 120 k 2023-03-27T10:12:41.263 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-mgr-dashboard noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 1.7 M 2023-03-27T10:12:41.263 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-mgr-diskprediction-local noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 7.4 M 2023-03-27T10:12:41.263 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-mgr-rook noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 47 k 2023-03-27T10:12:41.264 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-radosgw x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 13 M 2023-03-27T10:12:41.264 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-test x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 45 M 2023-03-27T10:12:41.264 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-volume noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 263 k 2023-03-27T10:12:41.264 INFO:teuthology.orchestra.run.smithi161.stdout: cephadm noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 82 k 2023-03-27T10:12:41.265 INFO:teuthology.orchestra.run.smithi161.stdout: cephfs-mirror x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 258 k 2023-03-27T10:12:41.265 INFO:teuthology.orchestra.run.smithi161.stdout: cephfs-top noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 18 k 2023-03-27T10:12:41.265 INFO:teuthology.orchestra.run.smithi161.stdout: elfutils-libelf-devel x86_64 0.186-1.el8 rhel-8-for-x86_64-baseos-rpms 60 k 2023-03-27T10:12:41.265 INFO:teuthology.orchestra.run.smithi161.stdout: flex x86_64 2.6.1-9.el8 rhel-8-for-x86_64-appstream-rpms 320 k 2023-03-27T10:12:41.266 INFO:teuthology.orchestra.run.smithi161.stdout: libaio-devel x86_64 0.3.112-1.el8 rhel-8-for-x86_64-baseos-rpms 19 k 2023-03-27T10:12:41.266 INFO:teuthology.orchestra.run.smithi161.stdout: libcephfs-devel x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 30 k 2023-03-27T10:12:41.266 INFO:teuthology.orchestra.run.smithi161.stdout: libcephfs2 x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 768 k 2023-03-27T10:12:41.266 INFO:teuthology.orchestra.run.smithi161.stdout: librados-devel x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 139 k 2023-03-27T10:12:41.266 INFO:teuthology.orchestra.run.smithi161.stdout: libtool x86_64 2.4.6-25.el8 rhel-8-for-x86_64-appstream-rpms 709 k 2023-03-27T10:12:41.266 INFO:teuthology.orchestra.run.smithi161.stdout: libuuid-devel x86_64 2.32.1-35.el8 rhel-8-for-x86_64-baseos-rpms 98 k 2023-03-27T10:12:41.266 INFO:teuthology.orchestra.run.smithi161.stdout: openssl-devel x86_64 1:1.1.1k-7.el8_6 rhel-8-for-x86_64-baseos-rpms 2.3 M 2023-03-27T10:12:41.266 INFO:teuthology.orchestra.run.smithi161.stdout: postgresql x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 1.5 M 2023-03-27T10:12:41.267 INFO:teuthology.orchestra.run.smithi161.stdout: postgresql-contrib x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 808 k 2023-03-27T10:12:41.267 INFO:teuthology.orchestra.run.smithi161.stdout: postgresql-server x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 5.1 M 2023-03-27T10:12:41.267 INFO:teuthology.orchestra.run.smithi161.stdout: python3-cephfs x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 190 k 2023-03-27T10:12:41.267 INFO:teuthology.orchestra.run.smithi161.stdout: python3-rados x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 371 k 2023-03-27T10:12:41.267 INFO:teuthology.orchestra.run.smithi161.stdout: python3-rbd x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 365 k 2023-03-27T10:12:41.267 INFO:teuthology.orchestra.run.smithi161.stdout: python3-rgw x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 107 k 2023-03-27T10:12:41.267 INFO:teuthology.orchestra.run.smithi161.stdout: rbd-fuse x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 95 k 2023-03-27T10:12:41.268 INFO:teuthology.orchestra.run.smithi161.stdout: rbd-mirror x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 3.4 M 2023-03-27T10:12:41.268 INFO:teuthology.orchestra.run.smithi161.stdout: rbd-nbd x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 180 k 2023-03-27T10:12:41.268 INFO:teuthology.orchestra.run.smithi161.stdout: xfsprogs-devel x86_64 5.0.0-10.el8 rhel-8-for-x86_64-baseos-rpms 72 k 2023-03-27T10:12:41.268 INFO:teuthology.orchestra.run.smithi161.stdout:Upgrading: 2023-03-27T10:12:41.268 INFO:teuthology.orchestra.run.smithi161.stdout: iproute x86_64 5.15.0-4.el8_6.1 rhel-8-for-x86_64-baseos-rpms 798 k 2023-03-27T10:12:41.268 INFO:teuthology.orchestra.run.smithi161.stdout: librados2 x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 3.6 M 2023-03-27T10:12:41.268 INFO:teuthology.orchestra.run.smithi161.stdout: librbd1 x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 3.6 M 2023-03-27T10:12:41.268 INFO:teuthology.orchestra.run.smithi161.stdout:Installing dependencies: 2023-03-27T10:12:41.269 INFO:teuthology.orchestra.run.smithi161.stdout: automake noarch 1.16.1-7.el8 rhel-8-for-x86_64-appstream-rpms 713 k 2023-03-27T10:12:41.269 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-common x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 21 M 2023-03-27T10:12:41.269 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-grafana-dashboards noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 23 k 2023-03-27T10:12:41.269 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-mds x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 2.3 M 2023-03-27T10:12:41.269 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-mgr-modules-core noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 236 k 2023-03-27T10:12:41.269 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-mon x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 3.9 M 2023-03-27T10:12:41.269 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-osd x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 17 M 2023-03-27T10:12:41.270 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-prometheus-alerts noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 15 k 2023-03-27T10:12:41.270 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-selinux x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 24 k 2023-03-27T10:12:41.270 INFO:teuthology.orchestra.run.smithi161.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-03-27T10:12:41.270 INFO:teuthology.orchestra.run.smithi161.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-03-27T10:12:41.270 INFO:teuthology.orchestra.run.smithi161.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-03-27T10:12:41.270 INFO:teuthology.orchestra.run.smithi161.stdout: keyutils-libs-devel x86_64 1.5.10-9.el8 rhel-8-for-x86_64-baseos-rpms 48 k 2023-03-27T10:12:41.270 INFO:teuthology.orchestra.run.smithi161.stdout: krb5-devel x86_64 1.18.2-14.el8 rhel-8-for-x86_64-baseos-rpms 560 k 2023-03-27T10:12:41.270 INFO:teuthology.orchestra.run.smithi161.stdout: libcephsqlite x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 137 k 2023-03-27T10:12:41.271 INFO:teuthology.orchestra.run.smithi161.stdout: libcom_err-devel x86_64 1.45.6-4.el8 rhel-8-for-x86_64-baseos-rpms 39 k 2023-03-27T10:12:41.271 INFO:teuthology.orchestra.run.smithi161.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-03-27T10:12:41.271 INFO:teuthology.orchestra.run.smithi161.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-03-27T10:12:41.271 INFO:teuthology.orchestra.run.smithi161.stdout: libpq x86_64 13.5-1.el8 rhel-8-for-x86_64-appstream-rpms 198 k 2023-03-27T10:12:41.271 INFO:teuthology.orchestra.run.smithi161.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-03-27T10:12:41.271 INFO:teuthology.orchestra.run.smithi161.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-03-27T10:12:41.271 INFO:teuthology.orchestra.run.smithi161.stdout: libradosstriper1 x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 474 k 2023-03-27T10:12:41.271 INFO:teuthology.orchestra.run.smithi161.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-03-27T10:12:41.272 INFO:teuthology.orchestra.run.smithi161.stdout: librgw2 x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 6.5 M 2023-03-27T10:12:41.272 INFO:teuthology.orchestra.run.smithi161.stdout: libselinux-devel x86_64 2.9-5.el8 rhel-8-for-x86_64-baseos-rpms 200 k 2023-03-27T10:12:41.272 INFO:teuthology.orchestra.run.smithi161.stdout: libsepol-devel x86_64 2.9-3.el8 rhel-8-for-x86_64-baseos-rpms 87 k 2023-03-27T10:12:41.272 INFO:teuthology.orchestra.run.smithi161.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-03-27T10:12:41.272 INFO:teuthology.orchestra.run.smithi161.stdout: libverto-devel x86_64 0.3.0-5.el8 rhel-8-for-x86_64-baseos-rpms 18 k 2023-03-27T10:12:41.272 INFO:teuthology.orchestra.run.smithi161.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-03-27T10:12:41.272 INFO:teuthology.orchestra.run.smithi161.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-03-27T10:12:41.273 INFO:teuthology.orchestra.run.smithi161.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-03-27T10:12:41.273 INFO:teuthology.orchestra.run.smithi161.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-03-27T10:12:41.273 INFO:teuthology.orchestra.run.smithi161.stdout: pcre2-devel x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 605 k 2023-03-27T10:12:41.273 INFO:teuthology.orchestra.run.smithi161.stdout: pcre2-utf16 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 229 k 2023-03-27T10:12:41.273 INFO:teuthology.orchestra.run.smithi161.stdout: pcre2-utf32 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 220 k 2023-03-27T10:12:41.273 INFO:teuthology.orchestra.run.smithi161.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-03-27T10:12:41.273 INFO:teuthology.orchestra.run.smithi161.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-03-27T10:12:41.273 INFO:teuthology.orchestra.run.smithi161.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-03-27T10:12:41.274 INFO:teuthology.orchestra.run.smithi161.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-03-27T10:12:41.274 INFO:teuthology.orchestra.run.smithi161.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-03-27T10:12:41.274 INFO:teuthology.orchestra.run.smithi161.stdout: python3-ceph-argparse x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 45 k 2023-03-27T10:12:41.274 INFO:teuthology.orchestra.run.smithi161.stdout: python3-ceph-common x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 117 k 2023-03-27T10:12:41.274 INFO:teuthology.orchestra.run.smithi161.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-03-27T10:12:41.274 INFO:teuthology.orchestra.run.smithi161.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-03-27T10:12:41.274 INFO:teuthology.orchestra.run.smithi161.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-03-27T10:12:41.275 INFO:teuthology.orchestra.run.smithi161.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-03-27T10:12:41.275 INFO:teuthology.orchestra.run.smithi161.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-03-27T10:12:41.275 INFO:teuthology.orchestra.run.smithi161.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-03-27T10:12:41.275 INFO:teuthology.orchestra.run.smithi161.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-03-27T10:12:41.275 INFO:teuthology.orchestra.run.smithi161.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-03-27T10:12:41.275 INFO:teuthology.orchestra.run.smithi161.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-03-27T10:12:41.275 INFO:teuthology.orchestra.run.smithi161.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-03-27T10:12:41.275 INFO:teuthology.orchestra.run.smithi161.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-03-27T10:12:41.276 INFO:teuthology.orchestra.run.smithi161.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-03-27T10:12:41.276 INFO:teuthology.orchestra.run.smithi161.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-03-27T10:12:41.276 INFO:teuthology.orchestra.run.smithi161.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-03-27T10:12:41.276 INFO:teuthology.orchestra.run.smithi161.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-03-27T10:12:41.276 INFO:teuthology.orchestra.run.smithi161.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-03-27T10:12:41.276 INFO:teuthology.orchestra.run.smithi161.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-03-27T10:12:41.276 INFO:teuthology.orchestra.run.smithi161.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-03-27T10:12:41.277 INFO:teuthology.orchestra.run.smithi161.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-03-27T10:12:41.277 INFO:teuthology.orchestra.run.smithi161.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-03-27T10:12:41.277 INFO:teuthology.orchestra.run.smithi161.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-03-27T10:12:41.277 INFO:teuthology.orchestra.run.smithi161.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-03-27T10:12:41.277 INFO:teuthology.orchestra.run.smithi161.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-03-27T10:12:41.277 INFO:teuthology.orchestra.run.smithi161.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-03-27T10:12:41.277 INFO:teuthology.orchestra.run.smithi161.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-03-27T10:12:41.277 INFO:teuthology.orchestra.run.smithi161.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-03-27T10:12:41.278 INFO:teuthology.orchestra.run.smithi161.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-03-27T10:12:41.278 INFO:teuthology.orchestra.run.smithi161.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-03-27T10:12:41.278 INFO:teuthology.orchestra.run.smithi161.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-03-27T10:12:41.278 INFO:teuthology.orchestra.run.smithi161.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-03-27T10:12:41.278 INFO:teuthology.orchestra.run.smithi161.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-03-27T10:12:41.278 INFO:teuthology.orchestra.run.smithi161.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-03-27T10:12:41.278 INFO:teuthology.orchestra.run.smithi161.stdout: python3-rsa noarch 4.8-1.el8 epel 61 k 2023-03-27T10:12:41.278 INFO:teuthology.orchestra.run.smithi161.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-03-27T10:12:41.279 INFO:teuthology.orchestra.run.smithi161.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-03-27T10:12:41.279 INFO:teuthology.orchestra.run.smithi161.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-03-27T10:12:41.279 INFO:teuthology.orchestra.run.smithi161.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-03-27T10:12:41.279 INFO:teuthology.orchestra.run.smithi161.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-03-27T10:12:41.279 INFO:teuthology.orchestra.run.smithi161.stdout: python3-waitress noarch 1.2.1-2.el8.1 epel 241 k 2023-03-27T10:12:41.279 INFO:teuthology.orchestra.run.smithi161.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-03-27T10:12:41.279 INFO:teuthology.orchestra.run.smithi161.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-03-27T10:12:41.279 INFO:teuthology.orchestra.run.smithi161.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-03-27T10:12:41.280 INFO:teuthology.orchestra.run.smithi161.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-03-27T10:12:41.280 INFO:teuthology.orchestra.run.smithi161.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-03-27T10:12:41.280 INFO:teuthology.orchestra.run.smithi161.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-03-27T10:12:41.280 INFO:teuthology.orchestra.run.smithi161.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-03-27T10:12:41.280 INFO:teuthology.orchestra.run.smithi161.stdout: uuid x86_64 1.6.2-43.el8 rhel-8-for-x86_64-appstream-rpms 64 k 2023-03-27T10:12:41.280 INFO:teuthology.orchestra.run.smithi161.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-03-27T10:12:41.280 INFO:teuthology.orchestra.run.smithi161.stdout: zlib-devel x86_64 1.2.11-18.el8_5 rhel-8-for-x86_64-baseos-rpms 58 k 2023-03-27T10:12:41.281 INFO:teuthology.orchestra.run.smithi161.stdout:Enabling module streams: 2023-03-27T10:12:41.281 INFO:teuthology.orchestra.run.smithi161.stdout: postgresql 10 2023-03-27T10:12:41.281 INFO:teuthology.orchestra.run.smithi161.stdout: 2023-03-27T10:12:41.281 INFO:teuthology.orchestra.run.smithi161.stdout:Transaction Summary 2023-03-27T10:12:41.281 INFO:teuthology.orchestra.run.smithi161.stdout:=========================================================================================================================== 2023-03-27T10:12:41.281 INFO:teuthology.orchestra.run.smithi161.stdout:Install 128 Packages 2023-03-27T10:12:41.281 INFO:teuthology.orchestra.run.smithi161.stdout:Upgrade 3 Packages 2023-03-27T10:12:41.281 INFO:teuthology.orchestra.run.smithi161.stdout: 2023-03-27T10:12:41.282 INFO:teuthology.orchestra.run.smithi161.stdout:Total download size: 199 M 2023-03-27T10:12:41.283 INFO:teuthology.orchestra.run.smithi161.stdout:Downloading Packages: 2023-03-27T10:12:41.283 INFO:teuthology.orchestra.run.smithi171.stdout:=========================================================================================================================== 2023-03-27T10:12:41.283 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2023-03-27T10:12:41.283 INFO:teuthology.orchestra.run.smithi171.stdout:=========================================================================================================================== 2023-03-27T10:12:41.283 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2023-03-27T10:12:41.283 INFO:teuthology.orchestra.run.smithi171.stdout: NetworkManager-initscripts-updown noarch 1:1.36.0-9.el8_6 rhel-8-for-x86_64-baseos-rpms 138 k 2023-03-27T10:12:41.284 INFO:teuthology.orchestra.run.smithi171.stdout: bison x86_64 3.0.4-10.el8 rhel-8-for-x86_64-appstream-rpms 688 k 2023-03-27T10:12:41.284 INFO:teuthology.orchestra.run.smithi171.stdout: ceph x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 6.4 k 2023-03-27T10:12:41.284 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-base x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 4.9 M 2023-03-27T10:12:41.284 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-fuse x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 845 k 2023-03-27T10:12:41.284 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-immutable-object-cache x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 165 k 2023-03-27T10:12:41.284 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 1.5 M 2023-03-27T10:12:41.284 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-cephadm noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 120 k 2023-03-27T10:12:41.284 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-dashboard noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 1.7 M 2023-03-27T10:12:41.285 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-diskprediction-local noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 7.4 M 2023-03-27T10:12:41.285 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-rook noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 47 k 2023-03-27T10:12:41.285 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-radosgw x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 13 M 2023-03-27T10:12:41.285 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-test x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 45 M 2023-03-27T10:12:41.285 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-volume noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 263 k 2023-03-27T10:12:41.285 INFO:teuthology.orchestra.run.smithi171.stdout: cephadm noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 82 k 2023-03-27T10:12:41.285 INFO:teuthology.orchestra.run.smithi171.stdout: cephfs-mirror x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 258 k 2023-03-27T10:12:41.286 INFO:teuthology.orchestra.run.smithi171.stdout: cephfs-top noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 18 k 2023-03-27T10:12:41.286 INFO:teuthology.orchestra.run.smithi171.stdout: elfutils-libelf-devel x86_64 0.186-1.el8 rhel-8-for-x86_64-baseos-rpms 60 k 2023-03-27T10:12:41.286 INFO:teuthology.orchestra.run.smithi171.stdout: flex x86_64 2.6.1-9.el8 rhel-8-for-x86_64-appstream-rpms 320 k 2023-03-27T10:12:41.286 INFO:teuthology.orchestra.run.smithi171.stdout: libaio-devel x86_64 0.3.112-1.el8 rhel-8-for-x86_64-baseos-rpms 19 k 2023-03-27T10:12:41.286 INFO:teuthology.orchestra.run.smithi171.stdout: libcephfs-devel x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 30 k 2023-03-27T10:12:41.286 INFO:teuthology.orchestra.run.smithi171.stdout: libcephfs2 x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 768 k 2023-03-27T10:12:41.286 INFO:teuthology.orchestra.run.smithi171.stdout: librados-devel x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 139 k 2023-03-27T10:12:41.286 INFO:teuthology.orchestra.run.smithi171.stdout: libtool x86_64 2.4.6-25.el8 rhel-8-for-x86_64-appstream-rpms 709 k 2023-03-27T10:12:41.287 INFO:teuthology.orchestra.run.smithi171.stdout: libuuid-devel x86_64 2.32.1-35.el8 rhel-8-for-x86_64-baseos-rpms 98 k 2023-03-27T10:12:41.287 INFO:teuthology.orchestra.run.smithi171.stdout: openssl-devel x86_64 1:1.1.1k-7.el8_6 rhel-8-for-x86_64-baseos-rpms 2.3 M 2023-03-27T10:12:41.287 INFO:teuthology.orchestra.run.smithi171.stdout: postgresql x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 1.5 M 2023-03-27T10:12:41.287 INFO:teuthology.orchestra.run.smithi171.stdout: postgresql-contrib x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 808 k 2023-03-27T10:12:41.287 INFO:teuthology.orchestra.run.smithi171.stdout: postgresql-server x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 5.1 M 2023-03-27T10:12:41.287 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cephfs x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 190 k 2023-03-27T10:12:41.287 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rados x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 371 k 2023-03-27T10:12:41.288 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rbd x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 365 k 2023-03-27T10:12:41.288 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rgw x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 107 k 2023-03-27T10:12:41.288 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-fuse x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 95 k 2023-03-27T10:12:41.288 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-mirror x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 3.4 M 2023-03-27T10:12:41.288 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-nbd x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 180 k 2023-03-27T10:12:41.288 INFO:teuthology.orchestra.run.smithi171.stdout: xfsprogs-devel x86_64 5.0.0-10.el8 rhel-8-for-x86_64-baseos-rpms 72 k 2023-03-27T10:12:41.288 INFO:teuthology.orchestra.run.smithi171.stdout:Upgrading: 2023-03-27T10:12:41.288 INFO:teuthology.orchestra.run.smithi171.stdout: iproute x86_64 5.15.0-4.el8_6.1 rhel-8-for-x86_64-baseos-rpms 798 k 2023-03-27T10:12:41.289 INFO:teuthology.orchestra.run.smithi171.stdout: librados2 x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 3.6 M 2023-03-27T10:12:41.289 INFO:teuthology.orchestra.run.smithi171.stdout: librbd1 x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 3.6 M 2023-03-27T10:12:41.289 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2023-03-27T10:12:41.289 INFO:teuthology.orchestra.run.smithi171.stdout: automake noarch 1.16.1-7.el8 rhel-8-for-x86_64-appstream-rpms 713 k 2023-03-27T10:12:41.289 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-common x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 21 M 2023-03-27T10:12:41.289 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-grafana-dashboards noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 23 k 2023-03-27T10:12:41.289 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mds x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 2.3 M 2023-03-27T10:12:41.289 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-modules-core noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 236 k 2023-03-27T10:12:41.290 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mon x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 3.9 M 2023-03-27T10:12:41.290 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-osd x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 17 M 2023-03-27T10:12:41.290 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-prometheus-alerts noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 15 k 2023-03-27T10:12:41.290 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-selinux x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 24 k 2023-03-27T10:12:41.290 INFO:teuthology.orchestra.run.smithi171.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-03-27T10:12:41.290 INFO:teuthology.orchestra.run.smithi171.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-03-27T10:12:41.290 INFO:teuthology.orchestra.run.smithi171.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-03-27T10:12:41.291 INFO:teuthology.orchestra.run.smithi171.stdout: keyutils-libs-devel x86_64 1.5.10-9.el8 rhel-8-for-x86_64-baseos-rpms 48 k 2023-03-27T10:12:41.291 INFO:teuthology.orchestra.run.smithi171.stdout: krb5-devel x86_64 1.18.2-14.el8 rhel-8-for-x86_64-baseos-rpms 560 k 2023-03-27T10:12:41.291 INFO:teuthology.orchestra.run.smithi171.stdout: libcephsqlite x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 137 k 2023-03-27T10:12:41.291 INFO:teuthology.orchestra.run.smithi171.stdout: libcom_err-devel x86_64 1.45.6-4.el8 rhel-8-for-x86_64-baseos-rpms 39 k 2023-03-27T10:12:41.291 INFO:teuthology.orchestra.run.smithi171.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-03-27T10:12:41.291 INFO:teuthology.orchestra.run.smithi171.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-03-27T10:12:41.291 INFO:teuthology.orchestra.run.smithi171.stdout: libpq x86_64 13.5-1.el8 rhel-8-for-x86_64-appstream-rpms 198 k 2023-03-27T10:12:41.291 INFO:teuthology.orchestra.run.smithi171.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-03-27T10:12:41.292 INFO:teuthology.orchestra.run.smithi171.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-03-27T10:12:41.292 INFO:teuthology.orchestra.run.smithi171.stdout: libradosstriper1 x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 474 k 2023-03-27T10:12:41.292 INFO:teuthology.orchestra.run.smithi171.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-03-27T10:12:41.292 INFO:teuthology.orchestra.run.smithi171.stdout: librgw2 x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 6.5 M 2023-03-27T10:12:41.292 INFO:teuthology.orchestra.run.smithi171.stdout: libselinux-devel x86_64 2.9-5.el8 rhel-8-for-x86_64-baseos-rpms 200 k 2023-03-27T10:12:41.292 INFO:teuthology.orchestra.run.smithi171.stdout: libsepol-devel x86_64 2.9-3.el8 rhel-8-for-x86_64-baseos-rpms 87 k 2023-03-27T10:12:41.292 INFO:teuthology.orchestra.run.smithi171.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-03-27T10:12:41.292 INFO:teuthology.orchestra.run.smithi171.stdout: libverto-devel x86_64 0.3.0-5.el8 rhel-8-for-x86_64-baseos-rpms 18 k 2023-03-27T10:12:41.293 INFO:teuthology.orchestra.run.smithi171.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-03-27T10:12:41.293 INFO:teuthology.orchestra.run.smithi171.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-03-27T10:12:41.293 INFO:teuthology.orchestra.run.smithi171.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-03-27T10:12:41.293 INFO:teuthology.orchestra.run.smithi171.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-03-27T10:12:41.293 INFO:teuthology.orchestra.run.smithi171.stdout: pcre2-devel x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 605 k 2023-03-27T10:12:41.293 INFO:teuthology.orchestra.run.smithi171.stdout: pcre2-utf16 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 229 k 2023-03-27T10:12:41.293 INFO:teuthology.orchestra.run.smithi171.stdout: pcre2-utf32 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 220 k 2023-03-27T10:12:41.294 INFO:teuthology.orchestra.run.smithi171.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-03-27T10:12:41.294 INFO:teuthology.orchestra.run.smithi171.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-03-27T10:12:41.294 INFO:teuthology.orchestra.run.smithi171.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-03-27T10:12:41.294 INFO:teuthology.orchestra.run.smithi171.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-03-27T10:12:41.294 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-03-27T10:12:41.294 INFO:teuthology.orchestra.run.smithi171.stdout: python3-ceph-argparse x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 45 k 2023-03-27T10:12:41.294 INFO:teuthology.orchestra.run.smithi171.stdout: python3-ceph-common x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 117 k 2023-03-27T10:12:41.294 INFO:teuthology.orchestra.run.smithi171.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-03-27T10:12:41.295 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-03-27T10:12:41.295 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-03-27T10:12:41.295 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-03-27T10:12:41.295 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-03-27T10:12:41.295 INFO:teuthology.orchestra.run.smithi171.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-03-27T10:12:41.295 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-03-27T10:12:41.295 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-03-27T10:12:41.295 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-03-27T10:12:41.296 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-03-27T10:12:41.296 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-03-27T10:12:41.296 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-03-27T10:12:41.296 INFO:teuthology.orchestra.run.smithi171.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-03-27T10:12:41.296 INFO:teuthology.orchestra.run.smithi171.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-03-27T10:12:41.296 INFO:teuthology.orchestra.run.smithi171.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-03-27T10:12:41.296 INFO:teuthology.orchestra.run.smithi171.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-03-27T10:12:41.296 INFO:teuthology.orchestra.run.smithi171.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-03-27T10:12:41.297 INFO:teuthology.orchestra.run.smithi171.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-03-27T10:12:41.297 INFO:teuthology.orchestra.run.smithi171.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-03-27T10:12:41.297 INFO:teuthology.orchestra.run.smithi171.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-03-27T10:12:41.297 INFO:teuthology.orchestra.run.smithi171.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-03-27T10:12:41.298 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-03-27T10:12:41.298 INFO:teuthology.orchestra.run.smithi171.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-03-27T10:12:41.298 INFO:teuthology.orchestra.run.smithi171.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-03-27T10:12:41.298 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-03-27T10:12:41.298 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-03-27T10:12:41.298 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-03-27T10:12:41.298 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-03-27T10:12:41.298 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-03-27T10:12:41.299 INFO:teuthology.orchestra.run.smithi171.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-03-27T10:12:41.299 INFO:teuthology.orchestra.run.smithi171.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-03-27T10:12:41.299 INFO:teuthology.orchestra.run.smithi171.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-03-27T10:12:41.299 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rsa noarch 4.8-1.el8 epel 61 k 2023-03-27T10:12:41.299 INFO:teuthology.orchestra.run.smithi171.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-03-27T10:12:41.299 INFO:teuthology.orchestra.run.smithi171.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-03-27T10:12:41.299 INFO:teuthology.orchestra.run.smithi171.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-03-27T10:12:41.299 INFO:teuthology.orchestra.run.smithi171.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-03-27T10:12:41.300 INFO:teuthology.orchestra.run.smithi171.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-03-27T10:12:41.300 INFO:teuthology.orchestra.run.smithi171.stdout: python3-waitress noarch 1.2.1-2.el8.1 epel 241 k 2023-03-27T10:12:41.300 INFO:teuthology.orchestra.run.smithi171.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-03-27T10:12:41.300 INFO:teuthology.orchestra.run.smithi171.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-03-27T10:12:41.300 INFO:teuthology.orchestra.run.smithi171.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-03-27T10:12:41.300 INFO:teuthology.orchestra.run.smithi171.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-03-27T10:12:41.300 INFO:teuthology.orchestra.run.smithi171.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-03-27T10:12:41.300 INFO:teuthology.orchestra.run.smithi171.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-03-27T10:12:41.301 INFO:teuthology.orchestra.run.smithi171.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-03-27T10:12:41.301 INFO:teuthology.orchestra.run.smithi171.stdout: uuid x86_64 1.6.2-43.el8 rhel-8-for-x86_64-appstream-rpms 64 k 2023-03-27T10:12:41.301 INFO:teuthology.orchestra.run.smithi171.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-03-27T10:12:41.301 INFO:teuthology.orchestra.run.smithi171.stdout: zlib-devel x86_64 1.2.11-18.el8_5 rhel-8-for-x86_64-baseos-rpms 58 k 2023-03-27T10:12:41.301 INFO:teuthology.orchestra.run.smithi171.stdout:Enabling module streams: 2023-03-27T10:12:41.302 INFO:teuthology.orchestra.run.smithi171.stdout: postgresql 10 2023-03-27T10:12:41.302 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:12:41.302 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2023-03-27T10:12:41.302 INFO:teuthology.orchestra.run.smithi171.stdout:=========================================================================================================================== 2023-03-27T10:12:41.302 INFO:teuthology.orchestra.run.smithi171.stdout:Install 128 Packages 2023-03-27T10:12:41.302 INFO:teuthology.orchestra.run.smithi171.stdout:Upgrade 3 Packages 2023-03-27T10:12:41.302 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:12:41.303 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 199 M 2023-03-27T10:12:41.303 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2023-03-27T10:12:41.344 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2023-03-27T10:12:41.356 INFO:teuthology.orchestra.run.smithi159.stdout:=========================================================================================================================== 2023-03-27T10:12:41.357 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repository Size 2023-03-27T10:12:41.357 INFO:teuthology.orchestra.run.smithi159.stdout:=========================================================================================================================== 2023-03-27T10:12:41.357 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2023-03-27T10:12:41.357 INFO:teuthology.orchestra.run.smithi159.stdout: NetworkManager-initscripts-updown noarch 1:1.36.0-9.el8_6 rhel-8-for-x86_64-baseos-rpms 138 k 2023-03-27T10:12:41.357 INFO:teuthology.orchestra.run.smithi159.stdout: bison x86_64 3.0.4-10.el8 rhel-8-for-x86_64-appstream-rpms 688 k 2023-03-27T10:12:41.357 INFO:teuthology.orchestra.run.smithi159.stdout: ceph x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 6.4 k 2023-03-27T10:12:41.358 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-base x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 4.9 M 2023-03-27T10:12:41.358 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-fuse x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 845 k 2023-03-27T10:12:41.358 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-immutable-object-cache x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 165 k 2023-03-27T10:12:41.358 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 1.5 M 2023-03-27T10:12:41.358 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-cephadm noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 120 k 2023-03-27T10:12:41.358 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-dashboard noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 1.7 M 2023-03-27T10:12:41.358 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-diskprediction-local noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 7.4 M 2023-03-27T10:12:41.359 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-rook noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 47 k 2023-03-27T10:12:41.359 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-radosgw x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 13 M 2023-03-27T10:12:41.359 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-test x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 45 M 2023-03-27T10:12:41.359 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-volume noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 263 k 2023-03-27T10:12:41.359 INFO:teuthology.orchestra.run.smithi159.stdout: cephadm noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 82 k 2023-03-27T10:12:41.359 INFO:teuthology.orchestra.run.smithi159.stdout: cephfs-mirror x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 258 k 2023-03-27T10:12:41.359 INFO:teuthology.orchestra.run.smithi159.stdout: cephfs-top noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 18 k 2023-03-27T10:12:41.359 INFO:teuthology.orchestra.run.smithi159.stdout: elfutils-libelf-devel x86_64 0.186-1.el8 rhel-8-for-x86_64-baseos-rpms 60 k 2023-03-27T10:12:41.360 INFO:teuthology.orchestra.run.smithi159.stdout: flex x86_64 2.6.1-9.el8 rhel-8-for-x86_64-appstream-rpms 320 k 2023-03-27T10:12:41.360 INFO:teuthology.orchestra.run.smithi159.stdout: libaio-devel x86_64 0.3.112-1.el8 rhel-8-for-x86_64-baseos-rpms 19 k 2023-03-27T10:12:41.360 INFO:teuthology.orchestra.run.smithi159.stdout: libcephfs-devel x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 30 k 2023-03-27T10:12:41.360 INFO:teuthology.orchestra.run.smithi159.stdout: libcephfs2 x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 768 k 2023-03-27T10:12:41.360 INFO:teuthology.orchestra.run.smithi159.stdout: librados-devel x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 139 k 2023-03-27T10:12:41.360 INFO:teuthology.orchestra.run.smithi159.stdout: libtool x86_64 2.4.6-25.el8 rhel-8-for-x86_64-appstream-rpms 709 k 2023-03-27T10:12:41.360 INFO:teuthology.orchestra.run.smithi159.stdout: libuuid-devel x86_64 2.32.1-35.el8 rhel-8-for-x86_64-baseos-rpms 98 k 2023-03-27T10:12:41.360 INFO:teuthology.orchestra.run.smithi159.stdout: openssl-devel x86_64 1:1.1.1k-7.el8_6 rhel-8-for-x86_64-baseos-rpms 2.3 M 2023-03-27T10:12:41.361 INFO:teuthology.orchestra.run.smithi159.stdout: postgresql x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 1.5 M 2023-03-27T10:12:41.361 INFO:teuthology.orchestra.run.smithi159.stdout: postgresql-contrib x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 808 k 2023-03-27T10:12:41.361 INFO:teuthology.orchestra.run.smithi159.stdout: postgresql-server x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 5.1 M 2023-03-27T10:12:41.361 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cephfs x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 190 k 2023-03-27T10:12:41.361 INFO:teuthology.orchestra.run.smithi159.stdout: python3-rados x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 371 k 2023-03-27T10:12:41.361 INFO:teuthology.orchestra.run.smithi159.stdout: python3-rbd x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 365 k 2023-03-27T10:12:41.361 INFO:teuthology.orchestra.run.smithi159.stdout: python3-rgw x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 107 k 2023-03-27T10:12:41.361 INFO:teuthology.orchestra.run.smithi159.stdout: rbd-fuse x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 95 k 2023-03-27T10:12:41.362 INFO:teuthology.orchestra.run.smithi159.stdout: rbd-mirror x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 3.4 M 2023-03-27T10:12:41.362 INFO:teuthology.orchestra.run.smithi159.stdout: rbd-nbd x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 180 k 2023-03-27T10:12:41.362 INFO:teuthology.orchestra.run.smithi159.stdout: xfsprogs-devel x86_64 5.0.0-10.el8 rhel-8-for-x86_64-baseos-rpms 72 k 2023-03-27T10:12:41.362 INFO:teuthology.orchestra.run.smithi159.stdout:Upgrading: 2023-03-27T10:12:41.362 INFO:teuthology.orchestra.run.smithi159.stdout: iproute x86_64 5.15.0-4.el8_6.1 rhel-8-for-x86_64-baseos-rpms 798 k 2023-03-27T10:12:41.362 INFO:teuthology.orchestra.run.smithi159.stdout: librados2 x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 3.6 M 2023-03-27T10:12:41.362 INFO:teuthology.orchestra.run.smithi159.stdout: librbd1 x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 3.6 M 2023-03-27T10:12:41.363 INFO:teuthology.orchestra.run.smithi159.stdout:Installing dependencies: 2023-03-27T10:12:41.363 INFO:teuthology.orchestra.run.smithi159.stdout: automake noarch 1.16.1-7.el8 rhel-8-for-x86_64-appstream-rpms 713 k 2023-03-27T10:12:41.363 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-common x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 21 M 2023-03-27T10:12:41.363 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-grafana-dashboards noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 23 k 2023-03-27T10:12:41.363 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mds x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 2.3 M 2023-03-27T10:12:41.363 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-modules-core noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 236 k 2023-03-27T10:12:41.363 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mon x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 3.9 M 2023-03-27T10:12:41.363 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-osd x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 17 M 2023-03-27T10:12:41.364 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-prometheus-alerts noarch 2:18.0.0-2533.gb70fdf97.el8 ceph-noarch 15 k 2023-03-27T10:12:41.364 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-selinux x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 24 k 2023-03-27T10:12:41.364 INFO:teuthology.orchestra.run.smithi159.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-03-27T10:12:41.364 INFO:teuthology.orchestra.run.smithi159.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-03-27T10:12:41.364 INFO:teuthology.orchestra.run.smithi159.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-03-27T10:12:41.364 INFO:teuthology.orchestra.run.smithi159.stdout: keyutils-libs-devel x86_64 1.5.10-9.el8 rhel-8-for-x86_64-baseos-rpms 48 k 2023-03-27T10:12:41.364 INFO:teuthology.orchestra.run.smithi159.stdout: krb5-devel x86_64 1.18.2-14.el8 rhel-8-for-x86_64-baseos-rpms 560 k 2023-03-27T10:12:41.364 INFO:teuthology.orchestra.run.smithi159.stdout: libcephsqlite x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 137 k 2023-03-27T10:12:41.365 INFO:teuthology.orchestra.run.smithi159.stdout: libcom_err-devel x86_64 1.45.6-4.el8 rhel-8-for-x86_64-baseos-rpms 39 k 2023-03-27T10:12:41.365 INFO:teuthology.orchestra.run.smithi159.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-03-27T10:12:41.365 INFO:teuthology.orchestra.run.smithi159.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-03-27T10:12:41.365 INFO:teuthology.orchestra.run.smithi159.stdout: libpq x86_64 13.5-1.el8 rhel-8-for-x86_64-appstream-rpms 198 k 2023-03-27T10:12:41.365 INFO:teuthology.orchestra.run.smithi159.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-03-27T10:12:41.366 INFO:teuthology.orchestra.run.smithi159.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-03-27T10:12:41.366 INFO:teuthology.orchestra.run.smithi159.stdout: libradosstriper1 x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 474 k 2023-03-27T10:12:41.366 INFO:teuthology.orchestra.run.smithi159.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-03-27T10:12:41.366 INFO:teuthology.orchestra.run.smithi159.stdout: librgw2 x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 6.5 M 2023-03-27T10:12:41.366 INFO:teuthology.orchestra.run.smithi159.stdout: libselinux-devel x86_64 2.9-5.el8 rhel-8-for-x86_64-baseos-rpms 200 k 2023-03-27T10:12:41.366 INFO:teuthology.orchestra.run.smithi159.stdout: libsepol-devel x86_64 2.9-3.el8 rhel-8-for-x86_64-baseos-rpms 87 k 2023-03-27T10:12:41.366 INFO:teuthology.orchestra.run.smithi159.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-03-27T10:12:41.367 INFO:teuthology.orchestra.run.smithi159.stdout: libverto-devel x86_64 0.3.0-5.el8 rhel-8-for-x86_64-baseos-rpms 18 k 2023-03-27T10:12:41.367 INFO:teuthology.orchestra.run.smithi159.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-03-27T10:12:41.367 INFO:teuthology.orchestra.run.smithi159.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-03-27T10:12:41.367 INFO:teuthology.orchestra.run.smithi159.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-03-27T10:12:41.367 INFO:teuthology.orchestra.run.smithi159.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-03-27T10:12:41.367 INFO:teuthology.orchestra.run.smithi159.stdout: pcre2-devel x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 605 k 2023-03-27T10:12:41.367 INFO:teuthology.orchestra.run.smithi159.stdout: pcre2-utf16 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 229 k 2023-03-27T10:12:41.367 INFO:teuthology.orchestra.run.smithi159.stdout: pcre2-utf32 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 220 k 2023-03-27T10:12:41.368 INFO:teuthology.orchestra.run.smithi159.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-03-27T10:12:41.368 INFO:teuthology.orchestra.run.smithi159.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-03-27T10:12:41.368 INFO:teuthology.orchestra.run.smithi159.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-03-27T10:12:41.368 INFO:teuthology.orchestra.run.smithi159.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-03-27T10:12:41.368 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-03-27T10:12:41.368 INFO:teuthology.orchestra.run.smithi159.stdout: python3-ceph-argparse x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 45 k 2023-03-27T10:12:41.369 INFO:teuthology.orchestra.run.smithi159.stdout: python3-ceph-common x86_64 2:18.0.0-2533.gb70fdf97.el8 ceph 117 k 2023-03-27T10:12:41.369 INFO:teuthology.orchestra.run.smithi159.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-03-27T10:12:41.369 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-03-27T10:12:41.369 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-03-27T10:12:41.369 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-03-27T10:12:41.369 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-03-27T10:12:41.369 INFO:teuthology.orchestra.run.smithi159.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-03-27T10:12:41.370 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-03-27T10:12:41.370 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-03-27T10:12:41.370 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-03-27T10:12:41.370 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-03-27T10:12:41.370 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-03-27T10:12:41.370 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-03-27T10:12:41.370 INFO:teuthology.orchestra.run.smithi159.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-03-27T10:12:41.370 INFO:teuthology.orchestra.run.smithi159.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-03-27T10:12:41.371 INFO:teuthology.orchestra.run.smithi159.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-03-27T10:12:41.371 INFO:teuthology.orchestra.run.smithi159.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-03-27T10:12:41.371 INFO:teuthology.orchestra.run.smithi159.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-03-27T10:12:41.371 INFO:teuthology.orchestra.run.smithi159.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-03-27T10:12:41.371 INFO:teuthology.orchestra.run.smithi159.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-03-27T10:12:41.371 INFO:teuthology.orchestra.run.smithi159.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-03-27T10:12:41.371 INFO:teuthology.orchestra.run.smithi159.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-03-27T10:12:41.371 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-03-27T10:12:41.372 INFO:teuthology.orchestra.run.smithi159.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-03-27T10:12:41.372 INFO:teuthology.orchestra.run.smithi159.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-03-27T10:12:41.372 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-03-27T10:12:41.372 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-03-27T10:12:41.372 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-03-27T10:12:41.372 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-03-27T10:12:41.372 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-03-27T10:12:41.372 INFO:teuthology.orchestra.run.smithi159.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-03-27T10:12:41.373 INFO:teuthology.orchestra.run.smithi159.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-03-27T10:12:41.373 INFO:teuthology.orchestra.run.smithi159.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-03-27T10:12:41.373 INFO:teuthology.orchestra.run.smithi159.stdout: python3-rsa noarch 4.8-1.el8 epel 61 k 2023-03-27T10:12:41.373 INFO:teuthology.orchestra.run.smithi159.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-03-27T10:12:41.373 INFO:teuthology.orchestra.run.smithi159.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-03-27T10:12:41.373 INFO:teuthology.orchestra.run.smithi159.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-03-27T10:12:41.374 INFO:teuthology.orchestra.run.smithi159.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-03-27T10:12:41.374 INFO:teuthology.orchestra.run.smithi159.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-03-27T10:12:41.374 INFO:teuthology.orchestra.run.smithi159.stdout: python3-waitress noarch 1.2.1-2.el8.1 epel 241 k 2023-03-27T10:12:41.374 INFO:teuthology.orchestra.run.smithi159.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-03-27T10:12:41.374 INFO:teuthology.orchestra.run.smithi159.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-03-27T10:12:41.374 INFO:teuthology.orchestra.run.smithi159.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-03-27T10:12:41.374 INFO:teuthology.orchestra.run.smithi159.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-03-27T10:12:41.375 INFO:teuthology.orchestra.run.smithi159.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-03-27T10:12:41.375 INFO:teuthology.orchestra.run.smithi159.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-03-27T10:12:41.375 INFO:teuthology.orchestra.run.smithi159.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-03-27T10:12:41.375 INFO:teuthology.orchestra.run.smithi159.stdout: uuid x86_64 1.6.2-43.el8 rhel-8-for-x86_64-appstream-rpms 64 k 2023-03-27T10:12:41.375 INFO:teuthology.orchestra.run.smithi159.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-03-27T10:12:41.375 INFO:teuthology.orchestra.run.smithi159.stdout: zlib-devel x86_64 1.2.11-18.el8_5 rhel-8-for-x86_64-baseos-rpms 58 k 2023-03-27T10:12:41.375 INFO:teuthology.orchestra.run.smithi159.stdout:Enabling module streams: 2023-03-27T10:12:41.376 INFO:teuthology.orchestra.run.smithi159.stdout: postgresql 10 2023-03-27T10:12:41.376 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:12:41.376 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2023-03-27T10:12:41.376 INFO:teuthology.orchestra.run.smithi159.stdout:=========================================================================================================================== 2023-03-27T10:12:41.376 INFO:teuthology.orchestra.run.smithi159.stdout:Install 128 Packages 2023-03-27T10:12:41.376 INFO:teuthology.orchestra.run.smithi159.stdout:Upgrade 3 Packages 2023-03-27T10:12:41.376 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:12:41.377 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 199 M 2023-03-27T10:12:41.377 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2023-03-27T10:12:41.392 INFO:teuthology.orchestra.run.smithi171.stdout:(1/131): ceph-18.0.0-2533.gb70fdf97.el8.x86_64. 57 kB/s | 6.4 kB 00:00 2023-03-27T10:12:41.403 INFO:teuthology.orchestra.run.smithi161.stdout:(1/131): ceph-18.0.0-2533.gb70fdf97.el8.x86_64. 51 kB/s | 6.4 kB 00:00 2023-03-27T10:12:41.497 INFO:teuthology.orchestra.run.smithi159.stdout:(1/131): ceph-18.0.0-2533.gb70fdf97.el8.x86_64. 52 kB/s | 6.4 kB 00:00 2023-03-27T10:12:41.892 INFO:teuthology.orchestra.run.smithi171.stdout:(2/131): ceph-fuse-18.0.0-2533.gb70fdf97.el8.x8 1.7 MB/s | 845 kB 00:00 2023-03-27T10:12:42.153 INFO:teuthology.orchestra.run.smithi161.stdout:(2/131): ceph-fuse-18.0.0-2533.gb70fdf97.el8.x8 1.1 MB/s | 845 kB 00:00 2023-03-27T10:12:42.220 INFO:teuthology.orchestra.run.smithi161.stdout:(3/131): ceph-base-18.0.0-2533.gb70fdf97.el8.x8 5.2 MB/s | 4.9 MB 00:00 2023-03-27T10:12:42.267 INFO:teuthology.orchestra.run.smithi171.stdout:(3/131): ceph-base-18.0.0-2533.gb70fdf97.el8.x8 4.9 MB/s | 4.9 MB 00:00 2023-03-27T10:12:42.278 INFO:teuthology.orchestra.run.smithi161.stdout:(4/131): ceph-immutable-object-cache-18.0.0-253 1.3 MB/s | 165 kB 00:00 2023-03-27T10:12:42.456 INFO:teuthology.orchestra.run.smithi159.stdout:(2/131): ceph-fuse-18.0.0-2533.gb70fdf97.el8.x8 882 kB/s | 845 kB 00:00 2023-03-27T10:12:42.493 INFO:teuthology.orchestra.run.smithi171.stdout:(4/131): ceph-immutable-object-cache-18.0.0-253 276 kB/s | 165 kB 00:00 2023-03-27T10:12:42.737 INFO:teuthology.orchestra.run.smithi161.stdout:(5/131): ceph-mds-18.0.0-2533.gb70fdf97.el8.x86 4.4 MB/s | 2.3 MB 00:00 2023-03-27T10:12:42.768 INFO:teuthology.orchestra.run.smithi171.stdout:(5/131): ceph-mds-18.0.0-2533.gb70fdf97.el8.x86 4.5 MB/s | 2.3 MB 00:00 2023-03-27T10:12:42.781 INFO:teuthology.orchestra.run.smithi159.stdout:(3/131): ceph-immutable-object-cache-18.0.0-253 509 kB/s | 165 kB 00:00 2023-03-27T10:12:43.296 INFO:teuthology.orchestra.run.smithi161.stdout:(6/131): ceph-mgr-18.0.0-2533.gb70fdf97.el8.x86 1.5 MB/s | 1.5 MB 00:01 2023-03-27T10:12:43.519 INFO:teuthology.orchestra.run.smithi171.stdout:(6/131): ceph-mon-18.0.0-2533.gb70fdf97.el8.x86 5.2 MB/s | 3.9 MB 00:00 2023-03-27T10:12:43.786 INFO:teuthology.orchestra.run.smithi161.stdout:(7/131): ceph-mon-18.0.0-2533.gb70fdf97.el8.x86 3.7 MB/s | 3.9 MB 00:01 2023-03-27T10:12:43.827 INFO:teuthology.orchestra.run.smithi171.stdout:(7/131): ceph-mgr-18.0.0-2533.gb70fdf97.el8.x86 1.2 MB/s | 1.5 MB 00:01 2023-03-27T10:12:44.294 INFO:teuthology.orchestra.run.smithi171.stdout:(8/131): ceph-common-18.0.0-2533.gb70fdf97.el8. 7.1 MB/s | 21 MB 00:03 2023-03-27T10:12:44.403 INFO:teuthology.orchestra.run.smithi171.stdout:(9/131): ceph-selinux-18.0.0-2533.gb70fdf97.el8 225 kB/s | 24 kB 00:00 2023-03-27T10:12:44.865 INFO:teuthology.orchestra.run.smithi159.stdout:(4/131): ceph-mds-18.0.0-2533.gb70fdf97.el8.x86 1.1 MB/s | 2.3 MB 00:02 2023-03-27T10:12:46.186 INFO:teuthology.orchestra.run.smithi171.stdout:(10/131): ceph-osd-18.0.0-2533.gb70fdf97.el8.x8 6.2 MB/s | 17 MB 00:02 2023-03-27T10:12:46.278 INFO:teuthology.orchestra.run.smithi171.stdout:(11/131): cephfs-mirror-18.0.0-2533.gb70fdf97.e 2.8 MB/s | 258 kB 00:00 2023-03-27T10:12:46.290 INFO:teuthology.orchestra.run.smithi159.stdout:(5/131): ceph-mgr-18.0.0-2533.gb70fdf97.el8.x86 1.1 MB/s | 1.5 MB 00:01 2023-03-27T10:12:46.320 INFO:teuthology.orchestra.run.smithi171.stdout:(12/131): libcephfs-devel-18.0.0-2533.gb70fdf97 731 kB/s | 30 kB 00:00 2023-03-27T10:12:46.365 INFO:teuthology.orchestra.run.smithi159.stdout:(6/131): ceph-base-18.0.0-2533.gb70fdf97.el8.x8 998 kB/s | 4.9 MB 00:04 2023-03-27T10:12:46.421 INFO:teuthology.orchestra.run.smithi161.stdout:(8/131): ceph-common-18.0.0-2533.gb70fdf97.el8. 4.2 MB/s | 21 MB 00:05 2023-03-27T10:12:46.428 INFO:teuthology.orchestra.run.smithi171.stdout:(13/131): libcephfs2-18.0.0-2533.gb70fdf97.el8. 6.9 MB/s | 768 kB 00:00 2023-03-27T10:12:46.495 INFO:teuthology.orchestra.run.smithi171.stdout:(14/131): libcephsqlite-18.0.0-2533.gb70fdf97.e 2.0 MB/s | 137 kB 00:00 2023-03-27T10:12:46.562 INFO:teuthology.orchestra.run.smithi171.stdout:(15/131): librados-devel-18.0.0-2533.gb70fdf97. 2.0 MB/s | 139 kB 00:00 2023-03-27T10:12:46.571 INFO:teuthology.orchestra.run.smithi161.stdout:(9/131): ceph-radosgw-18.0.0-2533.gb70fdf97.el8 4.7 MB/s | 13 MB 00:02 2023-03-27T10:12:46.597 INFO:teuthology.orchestra.run.smithi161.stdout:(10/131): ceph-selinux-18.0.0-2533.gb70fdf97.el 139 kB/s | 24 kB 00:00 2023-03-27T10:12:46.646 INFO:teuthology.orchestra.run.smithi171.stdout:(16/131): libradosstriper1-18.0.0-2533.gb70fdf9 5.6 MB/s | 474 kB 00:00 2023-03-27T10:12:46.755 INFO:teuthology.orchestra.run.smithi161.stdout:(11/131): cephfs-mirror-18.0.0-2533.gb70fdf97.e 1.6 MB/s | 258 kB 00:00 2023-03-27T10:12:46.856 INFO:teuthology.orchestra.run.smithi161.stdout:(12/131): libcephfs-devel-18.0.0-2533.gb70fdf97 305 kB/s | 30 kB 00:00 2023-03-27T10:12:47.189 INFO:teuthology.orchestra.run.smithi161.stdout:(13/131): libcephfs2-18.0.0-2533.gb70fdf97.el8. 2.3 MB/s | 768 kB 00:00 2023-03-27T10:12:47.290 INFO:teuthology.orchestra.run.smithi161.stdout:(14/131): libcephsqlite-18.0.0-2533.gb70fdf97.e 1.3 MB/s | 137 kB 00:00 2023-03-27T10:12:47.390 INFO:teuthology.orchestra.run.smithi161.stdout:(15/131): librados-devel-18.0.0-2533.gb70fdf97. 1.4 MB/s | 139 kB 00:00 2023-03-27T10:12:47.648 INFO:teuthology.orchestra.run.smithi161.stdout:(16/131): libradosstriper1-18.0.0-2533.gb70fdf9 1.8 MB/s | 474 kB 00:00 2023-03-27T10:12:47.688 INFO:teuthology.orchestra.run.smithi171.stdout:(17/131): librgw2-18.0.0-2533.gb70fdf97.el8.x86 6.2 MB/s | 6.5 MB 00:01 2023-03-27T10:12:47.763 INFO:teuthology.orchestra.run.smithi171.stdout:(18/131): python3-ceph-argparse-18.0.0-2533.gb7 596 kB/s | 45 kB 00:00 2023-03-27T10:12:47.855 INFO:teuthology.orchestra.run.smithi171.stdout:(19/131): python3-ceph-common-18.0.0-2533.gb70f 1.2 MB/s | 117 kB 00:00 2023-03-27T10:12:47.938 INFO:teuthology.orchestra.run.smithi171.stdout:(20/131): python3-cephfs-18.0.0-2533.gb70fdf97. 2.2 MB/s | 190 kB 00:00 2023-03-27T10:12:48.082 INFO:teuthology.orchestra.run.smithi159.stdout:(7/131): ceph-common-18.0.0-2533.gb70fdf97.el8. 3.2 MB/s | 21 MB 00:06 2023-03-27T10:12:48.123 INFO:teuthology.orchestra.run.smithi171.stdout:(21/131): python3-rados-18.0.0-2533.gb70fdf97.e 2.0 MB/s | 371 kB 00:00 2023-03-27T10:12:48.222 INFO:teuthology.orchestra.run.smithi171.stdout:(22/131): python3-rbd-18.0.0-2533.gb70fdf97.el8 3.6 MB/s | 365 kB 00:00 2023-03-27T10:12:48.314 INFO:teuthology.orchestra.run.smithi171.stdout:(23/131): python3-rgw-18.0.0-2533.gb70fdf97.el8 1.1 MB/s | 107 kB 00:00 2023-03-27T10:12:48.415 INFO:teuthology.orchestra.run.smithi171.stdout:(24/131): rbd-fuse-18.0.0-2533.gb70fdf97.el8.x8 946 kB/s | 95 kB 00:00 2023-03-27T10:12:49.000 INFO:teuthology.orchestra.run.smithi159.stdout:(8/131): ceph-mon-18.0.0-2533.gb70fdf97.el8.x86 1.4 MB/s | 3.9 MB 00:02 2023-03-27T10:12:49.099 INFO:teuthology.orchestra.run.smithi159.stdout:(9/131): ceph-selinux-18.0.0-2533.gb70fdf97.el8 244 kB/s | 24 kB 00:00 2023-03-27T10:12:49.182 INFO:teuthology.orchestra.run.smithi171.stdout:(25/131): rbd-mirror-18.0.0-2533.gb70fdf97.el8. 4.5 MB/s | 3.4 MB 00:00 2023-03-27T10:12:49.282 INFO:teuthology.orchestra.run.smithi171.stdout:(26/131): rbd-nbd-18.0.0-2533.gb70fdf97.el8.x86 1.8 MB/s | 180 kB 00:00 2023-03-27T10:12:49.382 INFO:teuthology.orchestra.run.smithi171.stdout:(27/131): ceph-grafana-dashboards-18.0.0-2533.g 231 kB/s | 23 kB 00:00 2023-03-27T10:12:49.482 INFO:teuthology.orchestra.run.smithi171.stdout:(28/131): ceph-mgr-cephadm-18.0.0-2533.gb70fdf9 1.2 MB/s | 120 kB 00:00 2023-03-27T10:12:50.149 INFO:teuthology.orchestra.run.smithi171.stdout:(29/131): ceph-mgr-dashboard-18.0.0-2533.gb70fd 2.6 MB/s | 1.7 MB 00:00 2023-03-27T10:12:50.732 INFO:teuthology.orchestra.run.smithi161.stdout:(17/131): librgw2-18.0.0-2533.gb70fdf97.el8.x86 2.1 MB/s | 6.5 MB 00:03 2023-03-27T10:12:50.832 INFO:teuthology.orchestra.run.smithi161.stdout:(18/131): python3-ceph-argparse-18.0.0-2533.gb7 447 kB/s | 45 kB 00:00 2023-03-27T10:12:51.041 INFO:teuthology.orchestra.run.smithi161.stdout:(19/131): ceph-osd-18.0.0-2533.gb70fdf97.el8.x8 2.1 MB/s | 17 MB 00:07 2023-03-27T10:12:51.066 INFO:teuthology.orchestra.run.smithi161.stdout:(20/131): python3-ceph-common-18.0.0-2533.gb70f 502 kB/s | 117 kB 00:00 2023-03-27T10:12:51.216 INFO:teuthology.orchestra.run.smithi161.stdout:(21/131): python3-cephfs-18.0.0-2533.gb70fdf97. 1.1 MB/s | 190 kB 00:00 2023-03-27T10:12:51.383 INFO:teuthology.orchestra.run.smithi161.stdout:(22/131): python3-rbd-18.0.0-2533.gb70fdf97.el8 2.1 MB/s | 365 kB 00:00 2023-03-27T10:12:51.475 INFO:teuthology.orchestra.run.smithi161.stdout:(23/131): python3-rgw-18.0.0-2533.gb70fdf97.el8 1.1 MB/s | 107 kB 00:00 2023-03-27T10:12:51.567 INFO:teuthology.orchestra.run.smithi161.stdout:(24/131): rbd-fuse-18.0.0-2533.gb70fdf97.el8.x8 1.0 MB/s | 95 kB 00:00 2023-03-27T10:12:51.592 INFO:teuthology.orchestra.run.smithi161.stdout:(25/131): python3-rados-18.0.0-2533.gb70fdf97.e 705 kB/s | 371 kB 00:00 2023-03-27T10:12:51.834 INFO:teuthology.orchestra.run.smithi161.stdout:(26/131): rbd-nbd-18.0.0-2533.gb70fdf97.el8.x86 743 kB/s | 180 kB 00:00 2023-03-27T10:12:51.926 INFO:teuthology.orchestra.run.smithi161.stdout:(27/131): ceph-grafana-dashboards-18.0.0-2533.g 252 kB/s | 23 kB 00:00 2023-03-27T10:12:52.085 INFO:teuthology.orchestra.run.smithi161.stdout:(28/131): ceph-mgr-cephadm-18.0.0-2533.gb70fdf9 761 kB/s | 120 kB 00:00 2023-03-27T10:12:52.658 INFO:teuthology.orchestra.run.smithi171.stdout:(30/131): ceph-test-18.0.0-2533.gb70fdf97.el8.x 5.5 MB/s | 45 MB 00:08 2023-03-27T10:12:52.781 INFO:teuthology.orchestra.run.smithi161.stdout:(29/131): rbd-mirror-18.0.0-2533.gb70fdf97.el8. 2.9 MB/s | 3.4 MB 00:01 2023-03-27T10:12:52.791 INFO:teuthology.orchestra.run.smithi171.stdout:(31/131): ceph-mgr-modules-core-18.0.0-2533.gb7 1.7 MB/s | 236 kB 00:00 2023-03-27T10:12:52.866 INFO:teuthology.orchestra.run.smithi159.stdout:(10/131): ceph-radosgw-18.0.0-2533.gb70fdf97.el 2.7 MB/s | 13 MB 00:04 2023-03-27T10:12:52.950 INFO:teuthology.orchestra.run.smithi171.stdout:(32/131): ceph-radosgw-18.0.0-2533.gb70fdf97.el 1.4 MB/s | 13 MB 00:09 2023-03-27T10:12:52.975 INFO:teuthology.orchestra.run.smithi171.stdout:(33/131): ceph-mgr-rook-18.0.0-2533.gb70fdf97.e 258 kB/s | 47 kB 00:00 2023-03-27T10:12:53.008 INFO:teuthology.orchestra.run.smithi159.stdout:(11/131): cephfs-mirror-18.0.0-2533.gb70fdf97.e 1.8 MB/s | 258 kB 00:00 2023-03-27T10:12:53.093 INFO:teuthology.orchestra.run.smithi171.stdout:(34/131): ceph-mgr-diskprediction-local-18.0.0- 2.5 MB/s | 7.4 MB 00:02 2023-03-27T10:12:53.100 INFO:teuthology.orchestra.run.smithi159.stdout:(12/131): libcephfs-devel-18.0.0-2533.gb70fdf97 333 kB/s | 30 kB 00:00 2023-03-27T10:12:53.118 INFO:teuthology.orchestra.run.smithi171.stdout:(35/131): ceph-prometheus-alerts-18.0.0-2533.gb 87 kB/s | 15 kB 00:00 2023-03-27T10:12:53.143 INFO:teuthology.orchestra.run.smithi171.stdout:(36/131): ceph-volume-18.0.0-2533.gb70fdf97.el8 1.5 MB/s | 263 kB 00:00 2023-03-27T10:12:53.168 INFO:teuthology.orchestra.run.smithi171.stdout:(37/131): cephadm-18.0.0-2533.gb70fdf97.el8.noa 1.1 MB/s | 82 kB 00:00 2023-03-27T10:12:53.273 INFO:teuthology.orchestra.run.smithi171.stdout:(38/131): cephfs-top-18.0.0-2533.gb70fdf97.el8. 116 kB/s | 18 kB 00:00 2023-03-27T10:12:53.293 INFO:teuthology.orchestra.run.smithi161.stdout:(30/131): ceph-mgr-dashboard-18.0.0-2533.gb70fd 1.4 MB/s | 1.7 MB 00:01 2023-03-27T10:12:53.350 INFO:teuthology.orchestra.run.smithi159.stdout:(13/131): libcephfs2-18.0.0-2533.gb70fdf97.el8. 3.0 MB/s | 768 kB 00:00 2023-03-27T10:12:53.417 INFO:teuthology.orchestra.run.smithi159.stdout:(14/131): libcephsqlite-18.0.0-2533.gb70fdf97.e 2.0 MB/s | 137 kB 00:00 2023-03-27T10:12:53.452 INFO:teuthology.orchestra.run.smithi161.stdout:(31/131): ceph-mgr-modules-core-18.0.0-2533.gb7 1.5 MB/s | 236 kB 00:00 2023-03-27T10:12:53.453 INFO:teuthology.orchestra.run.smithi171.stdout:(39/131): liboath-2.6.2-3.el8.x86_64.rpm 207 kB/s | 59 kB 00:00 2023-03-27T10:12:53.469 INFO:teuthology.orchestra.run.smithi171.stdout:(40/131): libunwind-1.3.1-3.el8.x86_64.rpm 383 kB/s | 75 kB 00:00 2023-03-27T10:12:53.501 INFO:teuthology.orchestra.run.smithi159.stdout:(15/131): librados-devel-18.0.0-2533.gb70fdf97. 1.6 MB/s | 139 kB 00:00 2023-03-27T10:12:53.502 INFO:teuthology.orchestra.run.smithi161.stdout:(32/131): ceph-mgr-rook-18.0.0-2533.gb70fdf97.e 954 kB/s | 47 kB 00:00 2023-03-27T10:12:53.520 INFO:teuthology.orchestra.run.smithi171.stdout:(41/131): gperftools-libs-2.7-9.el8.x86_64.rpm 814 kB/s | 306 kB 00:00 2023-03-27T10:12:53.545 INFO:teuthology.orchestra.run.smithi171.stdout:(42/131): python3-bcrypt-3.1.6-2.el8.1.x86_64.r 579 kB/s | 44 kB 00:00 2023-03-27T10:12:53.561 INFO:teuthology.orchestra.run.smithi161.stdout:(33/131): ceph-prometheus-alerts-18.0.0-2533.gb 248 kB/s | 15 kB 00:00 2023-03-27T10:12:53.595 INFO:teuthology.orchestra.run.smithi171.stdout:(43/131): python3-beautifulsoup4-4.6.3-2.el8.1. 2.4 MB/s | 185 kB 00:00 2023-03-27T10:12:53.620 INFO:teuthology.orchestra.run.smithi171.stdout:(44/131): python3-cachetools-3.1.1-4.el8.noarch 438 kB/s | 33 kB 00:00 2023-03-27T10:12:53.654 INFO:teuthology.orchestra.run.smithi171.stdout:(45/131): python3-asyncssh-2.7.0-2.el8.noarch.r 2.3 MB/s | 476 kB 00:00 2023-03-27T10:12:53.679 INFO:teuthology.orchestra.run.smithi171.stdout:(46/131): python3-certifi-2018.10.15-7.el8.noar 194 kB/s | 16 kB 00:00 2023-03-27T10:12:53.730 INFO:teuthology.orchestra.run.smithi171.stdout:(47/131): python3-cheroot-8.5.2-1.el8.noarch.rp 1.5 MB/s | 173 kB 00:00 2023-03-27T10:12:53.733 INFO:teuthology.orchestra.run.smithi159.stdout:(16/131): libradosstriper1-18.0.0-2533.gb70fdf9 2.0 MB/s | 474 kB 00:00 2023-03-27T10:12:53.753 INFO:teuthology.orchestra.run.smithi161.stdout:(34/131): ceph-volume-18.0.0-2533.gb70fdf97.el8 1.3 MB/s | 263 kB 00:00 2023-03-27T10:12:53.755 INFO:teuthology.orchestra.run.smithi171.stdout:(48/131): python3-google-auth-1.1.1-10.el8.noar 1.1 MB/s | 81 kB 00:00 2023-03-27T10:12:53.812 INFO:teuthology.orchestra.run.smithi161.stdout:(35/131): cephadm-18.0.0-2533.gb70fdf97.el8.noa 1.4 MB/s | 82 kB 00:00 2023-03-27T10:12:53.813 INFO:teuthology.orchestra.run.smithi171.stdout:(49/131): python3-cherrypy-18.4.0-1.el8.noarch. 2.4 MB/s | 384 kB 00:00 2023-03-27T10:12:53.847 INFO:teuthology.orchestra.run.smithi171.stdout:(50/131): python3-jaraco-6.2-6.el8.noarch.rpm 95 kB/s | 11 kB 00:00 2023-03-27T10:12:53.864 INFO:teuthology.orchestra.run.smithi171.stdout:(51/131): python3-jaraco-functools-2.0-4.el8.no 161 kB/s | 18 kB 00:00 2023-03-27T10:12:53.870 INFO:teuthology.orchestra.run.smithi161.stdout:(36/131): cephfs-top-18.0.0-2533.gb70fdf97.el8. 309 kB/s | 18 kB 00:00 2023-03-27T10:12:53.914 INFO:teuthology.orchestra.run.smithi171.stdout:(52/131): python3-logutils-0.3.5-11.el8.noarch. 729 kB/s | 49 kB 00:00 2023-03-27T10:12:53.939 INFO:teuthology.orchestra.run.smithi171.stdout:(53/131): python3-more-itertools-7.2.0-3.el8.no 780 kB/s | 59 kB 00:00 2023-03-27T10:12:53.981 INFO:teuthology.orchestra.run.smithi171.stdout:(54/131): python3-natsort-7.1.1-2.el8.noarch.rp 903 kB/s | 60 kB 00:00 2023-03-27T10:12:54.023 INFO:teuthology.orchestra.run.smithi171.stdout:(55/131): python3-kubernetes-11.0.0-6.el8.noarc 5.6 MB/s | 1.2 MB 00:00 2023-03-27T10:12:54.040 INFO:teuthology.orchestra.run.smithi171.stdout:(56/131): python3-portend-2.6-1.el8.noarch.rpm 266 kB/s | 16 kB 00:00 2023-03-27T10:12:54.062 INFO:teuthology.orchestra.run.smithi161.stdout:(37/131): gperftools-libs-2.7-9.el8.x86_64.rpm 1.6 MB/s | 306 kB 00:00 2023-03-27T10:12:54.066 INFO:teuthology.orchestra.run.smithi171.stdout:(57/131): python3-pecan-1.3.2-9.el8.noarch.rpm 2.2 MB/s | 283 kB 00:00 2023-03-27T10:12:54.082 INFO:teuthology.orchestra.run.smithi171.stdout:(58/131): python3-repoze-lru-0.7-6.el8.noarch.r 571 kB/s | 34 kB 00:00 2023-03-27T10:12:54.105 INFO:teuthology.orchestra.run.smithi161.stdout:(38/131): liboath-2.6.2-3.el8.x86_64.rpm 1.4 MB/s | 59 kB 00:00 2023-03-27T10:12:54.108 INFO:teuthology.orchestra.run.smithi171.stdout:(59/131): python3-routes-2.4.1-12.el8.noarch.rp 2.9 MB/s | 196 kB 00:00 2023-03-27T10:12:54.125 INFO:teuthology.orchestra.run.smithi171.stdout:(60/131): python3-rsa-4.8-1.el8.noarch.rpm 1.0 MB/s | 61 kB 00:00 2023-03-27T10:12:54.142 INFO:teuthology.orchestra.run.smithi171.stdout:(61/131): python3-simplegeneric-0.8.1-17.el8.no 318 kB/s | 19 kB 00:00 2023-03-27T10:12:54.155 INFO:teuthology.orchestra.run.smithi161.stdout:(39/131): libunwind-1.3.1-3.el8.x86_64.rpm 1.5 MB/s | 75 kB 00:00 2023-03-27T10:12:54.184 INFO:teuthology.orchestra.run.smithi171.stdout:(62/131): python3-trustme-0.6.0-4.el8.noarch.rp 656 kB/s | 27 kB 00:00 2023-03-27T10:12:54.218 INFO:teuthology.orchestra.run.smithi171.stdout:(63/131): python3-singledispatch-3.4.0.3-18.el8 218 kB/s | 24 kB 00:00 2023-03-27T10:12:54.234 INFO:teuthology.orchestra.run.smithi171.stdout:(64/131): python3-tempora-1.14.1-5.el8.noarch.r 267 kB/s | 29 kB 00:00 2023-03-27T10:12:54.260 INFO:teuthology.orchestra.run.smithi171.stdout:(65/131): python3-waitress-1.2.1-2.el8.1.noarch 3.1 MB/s | 241 kB 00:00 2023-03-27T10:12:54.264 INFO:teuthology.orchestra.run.smithi161.stdout:(40/131): python3-asyncssh-2.7.0-2.el8.noarch.r 4.3 MB/s | 476 kB 00:00 2023-03-27T10:12:54.285 INFO:teuthology.orchestra.run.smithi171.stdout:(66/131): python3-webob-1.8.5-1.el8.1.noarch.rp 3.7 MB/s | 251 kB 00:00 2023-03-27T10:12:54.310 INFO:teuthology.orchestra.run.smithi171.stdout:(67/131): python3-websocket-client-0.56.0-5.el8 811 kB/s | 61 kB 00:00 2023-03-27T10:12:54.322 INFO:teuthology.orchestra.run.smithi161.stdout:(41/131): python3-bcrypt-3.1.6-2.el8.1.x86_64.r 757 kB/s | 44 kB 00:00 2023-03-27T10:12:54.335 INFO:teuthology.orchestra.run.smithi171.stdout:(68/131): python3-webtest-2.0.33-1.el8.noarch.r 1.1 MB/s | 85 kB 00:00 2023-03-27T10:12:54.386 INFO:teuthology.orchestra.run.smithi171.stdout:(69/131): python3-zc-lockfile-2.0-2.el8.noarch. 228 kB/s | 23 kB 00:00 2023-03-27T10:12:54.411 INFO:teuthology.orchestra.run.smithi171.stdout:(70/131): xmlstarlet-1.6.1-20.el8.x86_64.rpm 925 kB/s | 70 kB 00:00 2023-03-27T10:12:54.453 INFO:teuthology.orchestra.run.smithi171.stdout:(71/131): librabbitmq-0.9.0-3.el8.x86_64.rpm 701 kB/s | 47 kB 00:00 2023-03-27T10:12:54.478 INFO:teuthology.orchestra.run.smithi171.stdout:(72/131): libselinux-devel-2.9-5.el8.x86_64.rpm 3.2 MB/s | 200 kB 00:00 2023-03-27T10:12:54.504 INFO:teuthology.orchestra.run.smithi171.stdout:(73/131): python3-jwt-1.6.1-2.el8.noarch.rpm 848 kB/s | 43 kB 00:00 2023-03-27T10:12:54.546 INFO:teuthology.orchestra.run.smithi171.stdout:(74/131): thrift-0.13.0-2.el8.x86_64.rpm 7.4 MB/s | 1.7 MB 00:00 2023-03-27T10:12:54.571 INFO:teuthology.orchestra.run.smithi171.stdout:(75/131): keyutils-libs-devel-1.5.10-9.el8.x86_ 719 kB/s | 48 kB 00:00 2023-03-27T10:12:54.581 INFO:teuthology.orchestra.run.smithi161.stdout:(42/131): ceph-mgr-diskprediction-local-18.0.0- 4.0 MB/s | 7.4 MB 00:01 2023-03-27T10:12:54.604 INFO:teuthology.orchestra.run.smithi171.stdout:(76/131): pcre2-utf32-10.32-2.el8.x86_64.rpm 1.7 MB/s | 220 kB 00:00 2023-03-27T10:12:54.630 INFO:teuthology.orchestra.run.smithi171.stdout:(77/131): python3-requests-oauthlib-1.0.0-1.el8 806 kB/s | 43 kB 00:00 2023-03-27T10:12:54.655 INFO:teuthology.orchestra.run.smithi171.stdout:(78/131): libsepol-devel-2.9-3.el8.x86_64.rpm 798 kB/s | 87 kB 00:00 2023-03-27T10:12:54.672 INFO:teuthology.orchestra.run.smithi171.stdout:(79/131): python3-pycparser-2.14-14.el8.noarch. 1.6 MB/s | 109 kB 00:00 2023-03-27T10:12:54.673 INFO:teuthology.orchestra.run.smithi161.stdout:(43/131): python3-beautifulsoup4-4.6.3-2.el8.1. 528 kB/s | 185 kB 00:00 2023-03-27T10:12:54.706 INFO:teuthology.orchestra.run.smithi171.stdout:(80/131): pcre2-utf16-10.32-2.el8.x86_64.rpm 3.0 MB/s | 229 kB 00:00 2023-03-27T10:12:54.723 INFO:teuthology.orchestra.run.smithi161.stdout:(44/131): python3-certifi-2018.10.15-7.el8.noar 327 kB/s | 16 kB 00:00 2023-03-27T10:12:54.731 INFO:teuthology.orchestra.run.smithi171.stdout:(81/131): libaio-devel-0.3.112-1.el8.x86_64.rpm 319 kB/s | 19 kB 00:00 2023-03-27T10:12:54.748 INFO:teuthology.orchestra.run.smithi161.stdout:(45/131): python3-cachetools-3.1.1-4.el8.noarch 197 kB/s | 33 kB 00:00 2023-03-27T10:12:54.756 INFO:teuthology.orchestra.run.smithi171.stdout:(82/131): python3-oauthlib-2.1.0-1.el8.noarch.r 3.0 MB/s | 155 kB 00:00 2023-03-27T10:12:54.790 INFO:teuthology.orchestra.run.smithi171.stdout:(83/131): krb5-devel-1.18.2-14.el8.x86_64.rpm 4.1 MB/s | 560 kB 00:00 2023-03-27T10:12:54.815 INFO:teuthology.orchestra.run.smithi171.stdout:(84/131): libverto-devel-0.3.0-5.el8.x86_64.rpm 305 kB/s | 18 kB 00:00 2023-03-27T10:12:54.815 INFO:teuthology.orchestra.run.smithi161.stdout:(46/131): python3-cheroot-8.5.2-1.el8.noarch.rp 1.8 MB/s | 173 kB 00:00 2023-03-27T10:12:54.849 INFO:teuthology.orchestra.run.smithi171.stdout:(85/131): python3-cryptography-3.2.1-5.el8.x86_ 4.7 MB/s | 559 kB 00:00 2023-03-27T10:12:54.865 INFO:teuthology.orchestra.run.smithi161.stdout:(47/131): python3-google-auth-1.1.1-10.el8.noar 1.6 MB/s | 81 kB 00:00 2023-03-27T10:12:54.874 INFO:teuthology.orchestra.run.smithi171.stdout:(86/131): python3-cffi-1.11.5-5.el8.x86_64.rpm 4.0 MB/s | 238 kB 00:00 2023-03-27T10:12:54.907 INFO:teuthology.orchestra.run.smithi171.stdout:(87/131): pcre2-devel-10.32-2.el8.x86_64.rpm 5.0 MB/s | 605 kB 00:00 2023-03-27T10:12:54.924 INFO:teuthology.orchestra.run.smithi161.stdout:(48/131): python3-cherrypy-18.4.0-1.el8.noarch. 2.1 MB/s | 384 kB 00:00 2023-03-27T10:12:54.949 INFO:teuthology.orchestra.run.smithi161.stdout:(49/131): python3-jaraco-6.2-6.el8.noarch.rpm 133 kB/s | 11 kB 00:00 2023-03-27T10:12:54.963 INFO:teuthology.orchestra.run.smithi171.stdout:(88/131): xfsprogs-devel-5.0.0-10.el8.x86_64.rp 811 kB/s | 72 kB 00:00 2023-03-27T10:12:54.975 INFO:teuthology.orchestra.run.smithi161.stdout:(50/131): python3-jaraco-functools-2.0-4.el8.no 351 kB/s | 18 kB 00:00 2023-03-27T10:12:54.991 INFO:teuthology.orchestra.run.smithi171.stdout:(89/131): fuse-2.9.7-15.el8.x86_64.rpm 990 kB/s | 83 kB 00:00 2023-03-27T10:12:55.016 INFO:teuthology.orchestra.run.smithi171.stdout:(90/131): elfutils-libelf-devel-0.186-1.el8.x86 1.1 MB/s | 60 kB 00:00 2023-03-27T10:12:55.025 INFO:teuthology.orchestra.run.smithi161.stdout:(51/131): python3-logutils-0.3.5-11.el8.noarch. 978 kB/s | 49 kB 00:00 2023-03-27T10:12:55.050 INFO:teuthology.orchestra.run.smithi171.stdout:(91/131): libuuid-devel-2.32.1-35.el8.x86_64.rp 1.6 MB/s | 98 kB 00:00 2023-03-27T10:12:55.075 INFO:teuthology.orchestra.run.smithi161.stdout:(52/131): python3-more-itertools-7.2.0-3.el8.no 1.1 MB/s | 59 kB 00:00 2023-03-27T10:12:55.076 INFO:teuthology.orchestra.run.smithi171.stdout:(92/131): zlib-devel-1.2.11-18.el8_5.x86_64.rpm 980 kB/s | 58 kB 00:00 2023-03-27T10:12:55.101 INFO:teuthology.orchestra.run.smithi171.stdout:(93/131): libcom_err-devel-1.45.6-4.el8.x86_64. 768 kB/s | 39 kB 00:00 2023-03-27T10:12:55.134 INFO:teuthology.orchestra.run.smithi161.stdout:(53/131): python3-kubernetes-11.0.0-6.el8.noarc 6.3 MB/s | 1.2 MB 00:00 2023-03-27T10:12:55.135 INFO:teuthology.orchestra.run.smithi171.stdout:(94/131): libgfortran-8.5.0-10.1.el8_6.x86_64.r 11 MB/s | 644 kB 00:00 2023-03-27T10:12:55.160 INFO:teuthology.orchestra.run.smithi171.stdout:(95/131): libquadmath-8.5.0-10.1.el8_6.x86_64.r 2.9 MB/s | 171 kB 00:00 2023-03-27T10:12:55.180 INFO:teuthology.orchestra.run.smithi161.stdout:(54/131): python3-natsort-7.1.1-2.el8.noarch.rp 575 kB/s | 60 kB 00:00 2023-03-27T10:12:55.185 INFO:teuthology.orchestra.run.smithi171.stdout:(96/131): NetworkManager-initscripts-updown-1.3 2.7 MB/s | 138 kB 00:00 2023-03-27T10:12:55.218 INFO:teuthology.orchestra.run.smithi161.stdout:(55/131): python3-pecan-1.3.2-9.el8.noarch.rpm 3.3 MB/s | 283 kB 00:00 2023-03-27T10:12:55.235 INFO:teuthology.orchestra.run.smithi161.stdout:(56/131): python3-portend-2.6-1.el8.noarch.rpm 289 kB/s | 16 kB 00:00 2023-03-27T10:12:55.244 INFO:teuthology.orchestra.run.smithi171.stdout:(97/131): openssl-devel-1.1.1k-7.el8_6.x86_64.r 5.9 MB/s | 2.3 MB 00:00 2023-03-27T10:12:55.260 INFO:teuthology.orchestra.run.smithi161.stdout:(57/131): python3-repoze-lru-0.7-6.el8.noarch.r 803 kB/s | 34 kB 00:00 2023-03-27T10:12:55.261 INFO:teuthology.orchestra.run.smithi171.stdout:(98/131): python3-pyasn1-modules-0.3.7-6.el8.no 1.4 MB/s | 110 kB 00:00 2023-03-27T10:12:55.277 INFO:teuthology.orchestra.run.smithi171.stdout:(99/131): python3-pytz-2017.2-9.el8.noarch.rpm 1.6 MB/s | 54 kB 00:00 2023-03-27T10:12:55.285 INFO:teuthology.orchestra.run.smithi161.stdout:(58/131): python3-routes-2.4.1-12.el8.noarch.rp 3.8 MB/s | 196 kB 00:00 2023-03-27T10:12:55.302 INFO:teuthology.orchestra.run.smithi161.stdout:(59/131): python3-rsa-4.8-1.el8.noarch.rpm 1.4 MB/s | 61 kB 00:00 2023-03-27T10:12:55.303 INFO:teuthology.orchestra.run.smithi171.stdout:(100/131): lttng-ust-2.8.1-11.el8.x86_64.rpm 6.1 MB/s | 259 kB 00:00 2023-03-27T10:12:55.328 INFO:teuthology.orchestra.run.smithi161.stdout:(60/131): python3-simplegeneric-0.8.1-17.el8.no 445 kB/s | 19 kB 00:00 2023-03-27T10:12:55.336 INFO:teuthology.orchestra.run.smithi171.stdout:(101/131): socat-1.7.4.1-1.el8.x86_64.rpm 5.4 MB/s | 323 kB 00:00 2023-03-27T10:12:55.345 INFO:teuthology.orchestra.run.smithi161.stdout:(61/131): python3-singledispatch-3.4.0.3-18.el8 567 kB/s | 24 kB 00:00 2023-03-27T10:12:55.362 INFO:teuthology.orchestra.run.smithi171.stdout:(102/131): python3-markupsafe-0.23-19.el8.x86_6 662 kB/s | 39 kB 00:00 2023-03-27T10:12:55.370 INFO:teuthology.orchestra.run.smithi161.stdout:(62/131): python3-tempora-1.14.1-5.el8.noarch.r 699 kB/s | 29 kB 00:00 2023-03-27T10:12:55.396 INFO:teuthology.orchestra.run.smithi161.stdout:(63/131): python3-trustme-0.6.0-4.el8.noarch.rp 547 kB/s | 27 kB 00:00 2023-03-27T10:12:55.396 INFO:teuthology.orchestra.run.smithi171.stdout:(103/131): bison-3.0.4-10.el8.x86_64.rpm 11 MB/s | 688 kB 00:00 2023-03-27T10:12:55.429 INFO:teuthology.orchestra.run.smithi171.stdout:(104/131): python3-jinja2-2.10.1-3.el8.noarch.r 7.9 MB/s | 538 kB 00:00 2023-03-27T10:12:55.430 INFO:teuthology.orchestra.run.smithi161.stdout:(64/131): python3-waitress-1.2.1-2.el8.1.noarch 4.0 MB/s | 241 kB 00:00 2023-03-27T10:12:55.455 INFO:teuthology.orchestra.run.smithi171.stdout:(105/131): python3-mako-1.0.6-13.el8.noarch.rpm 2.6 MB/s | 157 kB 00:00 2023-03-27T10:12:55.480 INFO:teuthology.orchestra.run.smithi161.stdout:(65/131): python3-webob-1.8.5-1.el8.1.noarch.rp 2.9 MB/s | 251 kB 00:00 2023-03-27T10:12:55.480 INFO:teuthology.orchestra.run.smithi171.stdout:(106/131): oniguruma-6.8.2-2.el8.x86_64.rpm 3.6 MB/s | 187 kB 00:00 2023-03-27T10:12:55.505 INFO:teuthology.orchestra.run.smithi161.stdout:(66/131): python3-websocket-client-0.56.0-5.el8 816 kB/s | 61 kB 00:00 2023-03-27T10:12:55.530 INFO:teuthology.orchestra.run.smithi161.stdout:(67/131): python3-webtest-2.0.33-1.el8.noarch.r 1.7 MB/s | 85 kB 00:00 2023-03-27T10:12:55.546 INFO:teuthology.orchestra.run.smithi171.stdout:(107/131): automake-1.16.1-7.el8.noarch.rpm 7.6 MB/s | 713 kB 00:00 2023-03-27T10:12:55.555 INFO:teuthology.orchestra.run.smithi161.stdout:(68/131): python3-zc-lockfile-2.0-2.el8.noarch. 456 kB/s | 23 kB 00:00 2023-03-27T10:12:55.580 INFO:teuthology.orchestra.run.smithi161.stdout:(69/131): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.8 MB/s | 70 kB 00:00 2023-03-27T10:12:55.588 INFO:teuthology.orchestra.run.smithi171.stdout:(108/131): python3-jsonpatch-1.21-2.el8.noarch. 639 kB/s | 27 kB 00:00 2023-03-27T10:12:55.601 INFO:teuthology.orchestra.run.smithi159.stdout:(17/131): librgw2-18.0.0-2533.gb70fdf97.el8.x86 3.5 MB/s | 6.5 MB 00:01 2023-03-27T10:12:55.639 INFO:teuthology.orchestra.run.smithi171.stdout:(109/131): flex-2.6.1-9.el8.x86_64.rpm 6.3 MB/s | 320 kB 00:00 2023-03-27T10:12:55.640 INFO:teuthology.orchestra.run.smithi161.stdout:(70/131): librabbitmq-0.9.0-3.el8.x86_64.rpm 803 kB/s | 47 kB 00:00 2023-03-27T10:12:55.676 INFO:teuthology.orchestra.run.smithi159.stdout:(18/131): python3-ceph-argparse-18.0.0-2533.gb7 597 kB/s | 45 kB 00:00 2023-03-27T10:12:55.681 INFO:teuthology.orchestra.run.smithi171.stdout:(110/131): python3-prettytable-0.7.2-14.el8.noa 1.0 MB/s | 44 kB 00:00 2023-03-27T10:12:55.681 INFO:teuthology.orchestra.run.smithi161.stdout:(71/131): libselinux-devel-2.9-5.el8.x86_64.rpm 4.7 MB/s | 200 kB 00:00 2023-03-27T10:12:55.756 INFO:teuthology.orchestra.run.smithi161.stdout:(72/131): python3-jwt-1.6.1-2.el8.noarch.rpm 575 kB/s | 43 kB 00:00 2023-03-27T10:12:55.765 INFO:teuthology.orchestra.run.smithi171.stdout:(111/131): python3-babel-2.5.1-7.el8.noarch.rpm 17 MB/s | 4.8 MB 00:00 2023-03-27T10:12:55.815 INFO:teuthology.orchestra.run.smithi171.stdout:(112/131): python3-werkzeug-0.12.2-4.el8.noarch 9.0 MB/s | 457 kB 00:00 2023-03-27T10:12:55.815 INFO:teuthology.orchestra.run.smithi161.stdout:(73/131): thrift-0.13.0-2.el8.x86_64.rpm 6.1 MB/s | 1.7 MB 00:00 2023-03-27T10:12:55.841 INFO:teuthology.orchestra.run.smithi161.stdout:(74/131): pcre2-utf32-10.32-2.el8.x86_64.rpm 2.6 MB/s | 220 kB 00:00 2023-03-27T10:12:55.843 INFO:teuthology.orchestra.run.smithi159.stdout:(19/131): ceph-osd-18.0.0-2533.gb70fdf97.el8.x8 1.7 MB/s | 17 MB 00:09 2023-03-27T10:12:55.865 INFO:teuthology.orchestra.run.smithi171.stdout:(113/131): python3-numpy-f2py-1.14.3-10.el8.x86 4.4 MB/s | 225 kB 00:00 2023-03-27T10:12:55.868 INFO:teuthology.orchestra.run.smithi159.stdout:(20/131): python3-ceph-common-18.0.0-2533.gb70f 610 kB/s | 117 kB 00:00 2023-03-27T10:12:55.891 INFO:teuthology.orchestra.run.smithi161.stdout:(75/131): keyutils-libs-devel-1.5.10-9.el8.x86_ 639 kB/s | 48 kB 00:00 2023-03-27T10:12:55.916 INFO:teuthology.orchestra.run.smithi161.stdout:(76/131): libsepol-devel-2.9-3.el8.x86_64.rpm 1.2 MB/s | 87 kB 00:00 2023-03-27T10:12:55.932 INFO:teuthology.orchestra.run.smithi171.stdout:(114/131): python3-numpy-1.14.3-10.el8.x86_64.r 15 MB/s | 3.7 MB 00:00 2023-03-27T10:12:55.936 INFO:teuthology.orchestra.run.smithi159.stdout:(21/131): python3-cephfs-18.0.0-2533.gb70fdf97. 2.0 MB/s | 190 kB 00:00 2023-03-27T10:12:55.941 INFO:teuthology.orchestra.run.smithi161.stdout:(77/131): python3-requests-oauthlib-1.0.0-1.el8 849 kB/s | 43 kB 00:00 2023-03-27T10:12:55.957 INFO:teuthology.orchestra.run.smithi171.stdout:(115/131): uuid-1.6.2-43.el8.x86_64.rpm 697 kB/s | 64 kB 00:00 2023-03-27T10:12:55.966 INFO:teuthology.orchestra.run.smithi161.stdout:(78/131): python3-pycparser-2.14-14.el8.noarch. 2.1 MB/s | 109 kB 00:00 2023-03-27T10:12:55.982 INFO:teuthology.orchestra.run.smithi171.stdout:(116/131): python3-pyasn1-0.3.7-6.el8.noarch.rp 2.5 MB/s | 126 kB 00:00 2023-03-27T10:12:55.992 INFO:teuthology.orchestra.run.smithi161.stdout:(79/131): pcre2-utf16-10.32-2.el8.x86_64.rpm 4.5 MB/s | 229 kB 00:00 2023-03-27T10:12:55.994 INFO:teuthology.orchestra.run.smithi159.stdout:(22/131): python3-rados-18.0.0-2533.gb70fdf97.e 2.9 MB/s | 371 kB 00:00 2023-03-27T10:12:56.008 INFO:teuthology.orchestra.run.smithi171.stdout:(117/131): python3-jsonpointer-1.10-11.el8.noar 396 kB/s | 20 kB 00:00 2023-03-27T10:12:56.033 INFO:teuthology.orchestra.run.smithi171.stdout:(118/131): python3-pyOpenSSL-19.0.0-1.el8.noarc 2.0 MB/s | 103 kB 00:00 2023-03-27T10:12:56.050 INFO:teuthology.orchestra.run.smithi161.stdout:(80/131): krb5-devel-1.18.2-14.el8.x86_64.rpm 6.5 MB/s | 560 kB 00:00 2023-03-27T10:12:56.061 INFO:teuthology.orchestra.run.smithi159.stdout:(23/131): python3-rgw-18.0.0-2533.gb70fdf97.el8 1.6 MB/s | 107 kB 00:00 2023-03-27T10:12:56.066 INFO:teuthology.orchestra.run.smithi171.stdout:(119/131): libtool-2.4.6-25.el8.x86_64.rpm 12 MB/s | 709 kB 00:00 2023-03-27T10:12:56.076 INFO:teuthology.orchestra.run.smithi161.stdout:(81/131): libaio-devel-0.3.112-1.el8.x86_64.rpm 224 kB/s | 19 kB 00:00 2023-03-27T10:12:56.087 INFO:teuthology.orchestra.run.smithi159.stdout:(24/131): python3-rbd-18.0.0-2533.gb70fdf97.el8 2.4 MB/s | 365 kB 00:00 2023-03-27T10:12:56.101 INFO:teuthology.orchestra.run.smithi161.stdout:(82/131): python3-oauthlib-2.1.0-1.el8.noarch.r 3.0 MB/s | 155 kB 00:00 2023-03-27T10:12:56.112 INFO:teuthology.orchestra.run.smithi159.stdout:(25/131): rbd-fuse-18.0.0-2533.gb70fdf97.el8.x8 1.8 MB/s | 95 kB 00:00 2023-03-27T10:12:56.135 INFO:teuthology.orchestra.run.smithi161.stdout:(83/131): python3-cryptography-3.2.1-5.el8.x86_ 9.3 MB/s | 559 kB 00:00 2023-03-27T10:12:56.160 INFO:teuthology.orchestra.run.smithi161.stdout:(84/131): libverto-devel-0.3.0-5.el8.x86_64.rpm 305 kB/s | 18 kB 00:00 2023-03-27T10:12:56.170 INFO:teuthology.orchestra.run.smithi159.stdout:(26/131): rbd-nbd-18.0.0-2533.gb70fdf97.el8.x86 3.0 MB/s | 180 kB 00:00 2023-03-27T10:12:56.193 INFO:teuthology.orchestra.run.smithi161.stdout:(85/131): pcre2-devel-10.32-2.el8.x86_64.rpm 10 MB/s | 605 kB 00:00 2023-03-27T10:12:56.209 INFO:teuthology.orchestra.run.smithi171.stdout:(120/131): python3-scipy-1.0.0-21.module+el8.5. 13 MB/s | 14 MB 00:01 2023-03-27T10:12:56.219 INFO:teuthology.orchestra.run.smithi161.stdout:(86/131): python3-cffi-1.11.5-5.el8.x86_64.rpm 4.0 MB/s | 238 kB 00:00 2023-03-27T10:12:56.221 INFO:teuthology.orchestra.run.smithi159.stdout:(27/131): ceph-grafana-dashboards-18.0.0-2533.g 461 kB/s | 23 kB 00:00 2023-03-27T10:12:56.234 INFO:teuthology.orchestra.run.smithi171.stdout:(121/131): libpq-13.5-1.el8.x86_64.rpm 1.2 MB/s | 198 kB 00:00 2023-03-27T10:12:56.261 INFO:teuthology.orchestra.run.smithi161.stdout:(87/131): xfsprogs-devel-5.0.0-10.el8.x86_64.rp 1.7 MB/s | 72 kB 00:00 2023-03-27T10:12:56.276 INFO:teuthology.orchestra.run.smithi171.stdout:(122/131): jq-1.6-3.el8.x86_64.rpm 4.7 MB/s | 202 kB 00:00 2023-03-27T10:12:56.279 INFO:teuthology.orchestra.run.smithi159.stdout:(28/131): ceph-mgr-cephadm-18.0.0-2533.gb70fdf9 2.0 MB/s | 120 kB 00:00 2023-03-27T10:12:56.318 INFO:teuthology.orchestra.run.smithi171.stdout:(123/131): librdkafka-0.11.4-3.el8.x86_64.rpm 8.3 MB/s | 354 kB 00:00 2023-03-27T10:12:56.345 INFO:teuthology.orchestra.run.smithi161.stdout:(88/131): openssl-devel-1.1.1k-7.el8_6.x86_64.r 15 MB/s | 2.3 MB 00:00 2023-03-27T10:12:56.370 INFO:teuthology.orchestra.run.smithi161.stdout:(89/131): fuse-2.9.7-15.el8.x86_64.rpm 761 kB/s | 83 kB 00:00 2023-03-27T10:12:56.395 INFO:teuthology.orchestra.run.smithi161.stdout:(90/131): elfutils-libelf-devel-0.186-1.el8.x86 1.2 MB/s | 60 kB 00:00 2023-03-27T10:12:56.410 INFO:teuthology.orchestra.run.smithi171.stdout:(124/131): openblas-threads-0.3.15-3.el8.x86_64 13 MB/s | 4.9 MB 00:00 2023-03-27T10:12:56.421 INFO:teuthology.orchestra.run.smithi161.stdout:(91/131): libuuid-devel-2.32.1-35.el8.x86_64.rp 1.9 MB/s | 98 kB 00:00 2023-03-27T10:12:56.446 INFO:teuthology.orchestra.run.smithi161.stdout:(92/131): zlib-devel-1.2.11-18.el8_5.x86_64.rpm 1.1 MB/s | 58 kB 00:00 2023-03-27T10:12:56.463 INFO:teuthology.orchestra.run.smithi161.stdout:(93/131): libcom_err-devel-1.45.6-4.el8.x86_64. 926 kB/s | 39 kB 00:00 2023-03-27T10:12:56.477 INFO:teuthology.orchestra.run.smithi171.stdout:(125/131): openblas-0.3.15-3.el8.x86_64.rpm 18 MB/s | 4.7 MB 00:00 2023-03-27T10:12:56.505 INFO:teuthology.orchestra.run.smithi161.stdout:(94/131): libgfortran-8.5.0-10.1.el8_6.x86_64.r 11 MB/s | 644 kB 00:00 2023-03-27T10:12:56.510 INFO:teuthology.orchestra.run.smithi171.stdout:(126/131): postgresql-contrib-10.21-2.module+el 7.9 MB/s | 808 kB 00:00 2023-03-27T10:12:56.530 INFO:teuthology.orchestra.run.smithi161.stdout:(95/131): libquadmath-8.5.0-10.1.el8_6.x86_64.r 2.5 MB/s | 171 kB 00:00 2023-03-27T10:12:56.555 INFO:teuthology.orchestra.run.smithi161.stdout:(96/131): NetworkManager-initscripts-updown-1.3 2.7 MB/s | 138 kB 00:00 2023-03-27T10:12:56.561 INFO:teuthology.orchestra.run.smithi171.stdout:(127/131): postgresql-10.21-2.module+el8.6.0+15 18 MB/s | 1.5 MB 00:00 2023-03-27T10:12:56.606 INFO:teuthology.orchestra.run.smithi161.stdout:(97/131): python3-pyasn1-modules-0.3.7-6.el8.no 2.2 MB/s | 110 kB 00:00 2023-03-27T10:12:56.639 INFO:teuthology.orchestra.run.smithi161.stdout:(98/131): python3-pytz-2017.2-9.el8.noarch.rpm 1.6 MB/s | 54 kB 00:00 2023-03-27T10:12:56.644 INFO:teuthology.orchestra.run.smithi171.stdout:(128/131): postgresql-server-10.21-2.module+el8 16 MB/s | 5.1 MB 00:00 2023-03-27T10:12:56.680 INFO:teuthology.orchestra.run.smithi159.stdout:(29/131): ceph-mgr-dashboard-18.0.0-2533.gb70fd 4.3 MB/s | 1.7 MB 00:00 2023-03-27T10:12:56.681 INFO:teuthology.orchestra.run.smithi161.stdout:(99/131): lttng-ust-2.8.1-11.el8.x86_64.rpm 6.1 MB/s | 259 kB 00:00 2023-03-27T10:12:56.703 INFO:teuthology.orchestra.run.smithi171.stdout:(129/131): iproute-5.15.0-4.el8_6.1.x86_64.rpm 13 MB/s | 798 kB 00:00 2023-03-27T10:12:56.723 INFO:teuthology.orchestra.run.smithi161.stdout:(100/131): socat-1.7.4.1-1.el8.x86_64.rpm 7.6 MB/s | 323 kB 00:00 2023-03-27T10:12:56.765 INFO:teuthology.orchestra.run.smithi161.stdout:(101/131): python3-markupsafe-0.23-19.el8.x86_6 936 kB/s | 39 kB 00:00 2023-03-27T10:12:56.832 INFO:teuthology.orchestra.run.smithi161.stdout:(102/131): bison-3.0.4-10.el8.x86_64.rpm 10 MB/s | 688 kB 00:00 2023-03-27T10:12:56.891 INFO:teuthology.orchestra.run.smithi161.stdout:(103/131): python3-jinja2-2.10.1-3.el8.noarch.r 9.0 MB/s | 538 kB 00:00 2023-03-27T10:12:56.924 INFO:teuthology.orchestra.run.smithi161.stdout:(104/131): python3-mako-1.0.6-13.el8.noarch.rpm 4.6 MB/s | 157 kB 00:00 2023-03-27T10:12:56.963 INFO:teuthology.orchestra.run.smithi159.stdout:(30/131): rbd-mirror-18.0.0-2533.gb70fdf97.el8. 3.9 MB/s | 3.4 MB 00:00 2023-03-27T10:12:57.058 INFO:teuthology.orchestra.run.smithi161.stdout:(105/131): python3-scipy-1.0.0-21.module+el8.5. 26 MB/s | 14 MB 00:00 2023-03-27T10:12:57.065 INFO:teuthology.orchestra.run.smithi159.stdout:(31/131): ceph-mgr-modules-core-18.0.0-2533.gb7 2.3 MB/s | 236 kB 00:00 2023-03-27T10:12:57.083 INFO:teuthology.orchestra.run.smithi161.stdout:(106/131): oniguruma-6.8.2-2.el8.x86_64.rpm 1.2 MB/s | 187 kB 00:00 2023-03-27T10:12:57.117 INFO:teuthology.orchestra.run.smithi161.stdout:(107/131): automake-1.16.1-7.el8.noarch.rpm 12 MB/s | 713 kB 00:00 2023-03-27T10:12:57.124 INFO:teuthology.orchestra.run.smithi159.stdout:(32/131): ceph-mgr-rook-18.0.0-2533.gb70fdf97.e 815 kB/s | 47 kB 00:00 2023-03-27T10:12:57.168 INFO:teuthology.orchestra.run.smithi161.stdout:(108/131): python3-jsonpatch-1.21-2.el8.noarch. 529 kB/s | 27 kB 00:00 2023-03-27T10:12:57.183 INFO:teuthology.orchestra.run.smithi159.stdout:(33/131): ceph-prometheus-alerts-18.0.0-2533.gb 249 kB/s | 15 kB 00:00 2023-03-27T10:12:57.209 INFO:teuthology.orchestra.run.smithi161.stdout:(109/131): flex-2.6.1-9.el8.x86_64.rpm 7.5 MB/s | 320 kB 00:00 2023-03-27T10:12:57.243 INFO:teuthology.orchestra.run.smithi161.stdout:(110/131): python3-prettytable-0.7.2-14.el8.noa 1.3 MB/s | 44 kB 00:00 2023-03-27T10:12:57.291 INFO:teuthology.orchestra.run.smithi159.stdout:(34/131): ceph-volume-18.0.0-2533.gb70fdf97.el8 2.4 MB/s | 263 kB 00:00 2023-03-27T10:12:57.310 INFO:teuthology.orchestra.run.smithi161.stdout:(111/131): python3-babel-2.5.1-7.el8.noarch.rpm 21 MB/s | 4.8 MB 00:00 2023-03-27T10:12:57.352 INFO:teuthology.orchestra.run.smithi161.stdout:(112/131): python3-werkzeug-0.12.2-4.el8.noarch 11 MB/s | 457 kB 00:00 2023-03-27T10:12:57.383 INFO:teuthology.orchestra.run.smithi159.stdout:(35/131): cephadm-18.0.0-2533.gb70fdf97.el8.noa 900 kB/s | 82 kB 00:00 2023-03-27T10:12:57.385 INFO:teuthology.orchestra.run.smithi161.stdout:(113/131): python3-numpy-f2py-1.14.3-10.el8.x86 6.6 MB/s | 225 kB 00:00 2023-03-27T10:12:57.434 INFO:teuthology.orchestra.run.smithi159.stdout:(36/131): cephfs-top-18.0.0-2533.gb70fdf97.el8. 360 kB/s | 18 kB 00:00 2023-03-27T10:12:57.461 INFO:teuthology.orchestra.run.smithi171.stdout:(130/131): librbd1-18.0.0-2533.gb70fdf97.el8.x8 4.0 MB/s | 3.6 MB 00:00 2023-03-27T10:12:57.469 INFO:teuthology.orchestra.run.smithi161.stdout:(114/131): python3-numpy-1.14.3-10.el8.x86_64.r 17 MB/s | 3.7 MB 00:00 2023-03-27T10:12:57.494 INFO:teuthology.orchestra.run.smithi161.stdout:(115/131): uuid-1.6.2-43.el8.x86_64.rpm 589 kB/s | 64 kB 00:00 2023-03-27T10:12:57.520 INFO:teuthology.orchestra.run.smithi161.stdout:(116/131): python3-pyasn1-0.3.7-6.el8.noarch.rp 2.4 MB/s | 126 kB 00:00 2023-03-27T10:12:57.545 INFO:teuthology.orchestra.run.smithi161.stdout:(117/131): python3-jsonpointer-1.10-11.el8.noar 396 kB/s | 20 kB 00:00 2023-03-27T10:12:57.570 INFO:teuthology.orchestra.run.smithi161.stdout:(118/131): python3-pyOpenSSL-19.0.0-1.el8.noarc 2.0 MB/s | 103 kB 00:00 2023-03-27T10:12:57.604 INFO:teuthology.orchestra.run.smithi161.stdout:(119/131): libtool-2.4.6-25.el8.x86_64.rpm 12 MB/s | 709 kB 00:00 2023-03-27T10:12:57.628 INFO:teuthology.orchestra.run.smithi171.stdout:(131/131): librados2-18.0.0-2533.gb70fdf97.el8. 3.3 MB/s | 3.6 MB 00:01 2023-03-27T10:12:57.631 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2023-03-27T10:12:57.631 INFO:teuthology.orchestra.run.smithi171.stdout:Total 12 MB/s | 199 MB 00:16 2023-03-27T10:12:57.634 INFO:teuthology.orchestra.run.smithi159.stdout:(37/131): gperftools-libs-2.7-9.el8.x86_64.rpm 1.5 MB/s | 306 kB 00:00 2023-03-27T10:12:57.646 INFO:teuthology.orchestra.run.smithi161.stdout:(120/131): libpq-13.5-1.el8.x86_64.rpm 4.6 MB/s | 198 kB 00:00 2023-03-27T10:12:57.684 INFO:teuthology.orchestra.run.smithi159.stdout:(38/131): liboath-2.6.2-3.el8.x86_64.rpm 1.2 MB/s | 59 kB 00:00 2023-03-27T10:12:57.735 INFO:teuthology.orchestra.run.smithi159.stdout:(39/131): libunwind-1.3.1-3.el8.x86_64.rpm 1.5 MB/s | 75 kB 00:00 2023-03-27T10:12:57.755 INFO:teuthology.orchestra.run.smithi161.stdout:(121/131): openblas-threads-0.3.15-3.el8.x86_64 26 MB/s | 4.9 MB 00:00 2023-03-27T10:12:57.796 INFO:teuthology.orchestra.run.smithi161.stdout:(122/131): jq-1.6-3.el8.x86_64.rpm 4.8 MB/s | 202 kB 00:00 2023-03-27T10:12:57.844 INFO:teuthology.orchestra.run.smithi159.stdout:(40/131): python3-asyncssh-2.7.0-2.el8.noarch.r 4.3 MB/s | 476 kB 00:00 2023-03-27T10:12:57.872 INFO:teuthology.orchestra.run.smithi161.stdout:(123/131): openblas-0.3.15-3.el8.x86_64.rpm 21 MB/s | 4.7 MB 00:00 2023-03-27T10:12:57.885 INFO:teuthology.orchestra.run.smithi159.stdout:(41/131): python3-bcrypt-3.1.6-2.el8.1.x86_64.r 1.0 MB/s | 44 kB 00:00 2023-03-27T10:12:57.897 INFO:teuthology.orchestra.run.smithi161.stdout:(124/131): librdkafka-0.11.4-3.el8.x86_64.rpm 3.4 MB/s | 354 kB 00:00 2023-03-27T10:12:57.944 INFO:teuthology.orchestra.run.smithi159.stdout:(42/131): python3-beautifulsoup4-4.6.3-2.el8.1. 3.1 MB/s | 185 kB 00:00 2023-03-27T10:12:57.947 INFO:teuthology.orchestra.run.smithi161.stdout:(125/131): postgresql-contrib-10.21-2.module+el 16 MB/s | 808 kB 00:00 2023-03-27T10:12:57.986 INFO:teuthology.orchestra.run.smithi159.stdout:(43/131): python3-cachetools-3.1.1-4.el8.noarch 793 kB/s | 33 kB 00:00 2023-03-27T10:12:58.039 INFO:teuthology.orchestra.run.smithi161.stdout:(126/131): postgresql-server-10.21-2.module+el8 30 MB/s | 5.1 MB 00:00 2023-03-27T10:12:58.098 INFO:teuthology.orchestra.run.smithi161.stdout:(127/131): postgresql-10.21-2.module+el8.6.0+15 10 MB/s | 1.5 MB 00:00 2023-03-27T10:12:58.120 INFO:teuthology.orchestra.run.smithi159.stdout:(44/131): ceph-mgr-diskprediction-local-18.0.0- 5.1 MB/s | 7.4 MB 00:01 2023-03-27T10:12:58.137 INFO:teuthology.orchestra.run.smithi159.stdout:(45/131): python3-certifi-2018.10.15-7.el8.noar 108 kB/s | 16 kB 00:00 2023-03-27T10:12:58.220 INFO:teuthology.orchestra.run.smithi159.stdout:(46/131): python3-cherrypy-18.4.0-1.el8.noarch. 4.5 MB/s | 384 kB 00:00 2023-03-27T10:12:58.271 INFO:teuthology.orchestra.run.smithi159.stdout:(47/131): python3-google-auth-1.1.1-10.el8.noar 1.6 MB/s | 81 kB 00:00 2023-03-27T10:12:58.317 INFO:teuthology.orchestra.run.smithi159.stdout:(48/131): python3-cheroot-8.5.2-1.el8.noarch.rp 879 kB/s | 173 kB 00:00 2023-03-27T10:12:58.346 INFO:teuthology.orchestra.run.smithi159.stdout:(49/131): python3-jaraco-6.2-6.el8.noarch.rpm 147 kB/s | 11 kB 00:00 2023-03-27T10:12:58.372 INFO:teuthology.orchestra.run.smithi159.stdout:(50/131): python3-jaraco-functools-2.0-4.el8.no 322 kB/s | 18 kB 00:00 2023-03-27T10:12:58.422 INFO:teuthology.orchestra.run.smithi159.stdout:(51/131): python3-logutils-0.3.5-11.el8.noarch. 979 kB/s | 49 kB 00:00 2023-03-27T10:12:58.459 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2023-03-27T10:12:58.472 INFO:teuthology.orchestra.run.smithi159.stdout:(52/131): python3-more-itertools-7.2.0-3.el8.no 1.1 MB/s | 59 kB 00:00 2023-03-27T10:12:58.515 INFO:teuthology.orchestra.run.smithi161.stdout:(128/131): librados2-18.0.0-2533.gb70fdf97.el8. 7.7 MB/s | 3.6 MB 00:00 2023-03-27T10:12:58.522 INFO:teuthology.orchestra.run.smithi159.stdout:(53/131): python3-natsort-7.1.1-2.el8.noarch.rp 1.2 MB/s | 60 kB 00:00 2023-03-27T10:12:58.578 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2023-03-27T10:12:58.579 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2023-03-27T10:12:58.582 INFO:teuthology.orchestra.run.smithi161.stdout:(129/131): iproute-5.15.0-4.el8_6.1.x86_64.rpm 12 MB/s | 798 kB 00:00 2023-03-27T10:12:58.589 INFO:teuthology.orchestra.run.smithi159.stdout:(54/131): python3-kubernetes-11.0.0-6.el8.noarc 4.8 MB/s | 1.2 MB 00:00 2023-03-27T10:12:58.615 INFO:teuthology.orchestra.run.smithi159.stdout:(55/131): python3-pecan-1.3.2-9.el8.noarch.rpm 3.0 MB/s | 283 kB 00:00 2023-03-27T10:12:58.640 INFO:teuthology.orchestra.run.smithi159.stdout:(56/131): python3-portend-2.6-1.el8.noarch.rpm 311 kB/s | 16 kB 00:00 2023-03-27T10:12:58.665 INFO:teuthology.orchestra.run.smithi159.stdout:(57/131): python3-repoze-lru-0.7-6.el8.noarch.r 671 kB/s | 34 kB 00:00 2023-03-27T10:12:58.707 INFO:teuthology.orchestra.run.smithi159.stdout:(58/131): python3-routes-2.4.1-12.el8.noarch.rp 2.9 MB/s | 196 kB 00:00 2023-03-27T10:12:58.733 INFO:teuthology.orchestra.run.smithi159.stdout:(59/131): python3-rsa-4.8-1.el8.noarch.rpm 917 kB/s | 61 kB 00:00 2023-03-27T10:12:58.758 INFO:teuthology.orchestra.run.smithi159.stdout:(60/131): python3-simplegeneric-0.8.1-17.el8.no 372 kB/s | 19 kB 00:00 2023-03-27T10:12:58.816 INFO:teuthology.orchestra.run.smithi159.stdout:(61/131): python3-singledispatch-3.4.0.3-18.el8 289 kB/s | 24 kB 00:00 2023-03-27T10:12:58.850 INFO:teuthology.orchestra.run.smithi159.stdout:(62/131): python3-tempora-1.14.1-5.el8.noarch.r 318 kB/s | 29 kB 00:00 2023-03-27T10:12:58.875 INFO:teuthology.orchestra.run.smithi159.stdout:(63/131): python3-trustme-0.6.0-4.el8.noarch.rp 463 kB/s | 27 kB 00:00 2023-03-27T10:12:58.909 INFO:teuthology.orchestra.run.smithi159.stdout:(64/131): python3-waitress-1.2.1-2.el8.1.noarch 4.0 MB/s | 241 kB 00:00 2023-03-27T10:12:58.959 INFO:teuthology.orchestra.run.smithi159.stdout:(65/131): python3-webob-1.8.5-1.el8.1.noarch.rp 2.9 MB/s | 251 kB 00:00 2023-03-27T10:12:58.985 INFO:teuthology.orchestra.run.smithi159.stdout:(66/131): python3-websocket-client-0.56.0-5.el8 815 kB/s | 61 kB 00:00 2023-03-27T10:12:59.010 INFO:teuthology.orchestra.run.smithi159.stdout:(67/131): python3-webtest-2.0.33-1.el8.noarch.r 1.7 MB/s | 85 kB 00:00 2023-03-27T10:12:59.077 INFO:teuthology.orchestra.run.smithi159.stdout:(68/131): python3-zc-lockfile-2.0-2.el8.noarch. 249 kB/s | 23 kB 00:00 2023-03-27T10:12:59.094 INFO:teuthology.orchestra.run.smithi159.stdout:(69/131): xmlstarlet-1.6.1-20.el8.x86_64.rpm 4.1 MB/s | 70 kB 00:00 2023-03-27T10:12:59.332 INFO:teuthology.orchestra.run.smithi161.stdout:(130/131): ceph-test-18.0.0-2533.gb70fdf97.el8. 3.5 MB/s | 45 MB 00:12 2023-03-27T10:12:59.382 INFO:teuthology.orchestra.run.smithi161.stdout:(131/131): librbd1-18.0.0-2533.gb70fdf97.el8.x8 2.8 MB/s | 3.6 MB 00:01 2023-03-27T10:12:59.385 INFO:teuthology.orchestra.run.smithi161.stdout:-------------------------------------------------------------------------------- 2023-03-27T10:12:59.386 INFO:teuthology.orchestra.run.smithi161.stdout:Total 11 MB/s | 199 MB 00:18 2023-03-27T10:12:59.394 INFO:teuthology.orchestra.run.smithi159.stdout:(70/131): python3-scipy-1.0.0-21.module+el8.5.0 46 MB/s | 14 MB 00:00 2023-03-27T10:12:59.436 INFO:teuthology.orchestra.run.smithi159.stdout:(71/131): thrift-0.13.0-2.el8.x86_64.rpm 4.1 MB/s | 1.7 MB 00:00 2023-03-27T10:12:59.453 INFO:teuthology.orchestra.run.smithi159.stdout:(72/131): python3-pyasn1-modules-0.3.7-6.el8.no 1.8 MB/s | 110 kB 00:00 2023-03-27T10:12:59.495 INFO:teuthology.orchestra.run.smithi159.stdout:(73/131): python3-pytz-2017.2-9.el8.noarch.rpm 926 kB/s | 54 kB 00:00 2023-03-27T10:12:59.546 INFO:teuthology.orchestra.run.smithi159.stdout:(74/131): lttng-ust-2.8.1-11.el8.x86_64.rpm 2.9 MB/s | 259 kB 00:00 2023-03-27T10:12:59.571 INFO:teuthology.orchestra.run.smithi159.stdout:(75/131): socat-1.7.4.1-1.el8.x86_64.rpm 4.2 MB/s | 323 kB 00:00 2023-03-27T10:12:59.588 INFO:teuthology.orchestra.run.smithi159.stdout:(76/131): python3-markupsafe-0.23-19.el8.x86_64 928 kB/s | 39 kB 00:00 2023-03-27T10:12:59.621 INFO:teuthology.orchestra.run.smithi159.stdout:(77/131): bison-3.0.4-10.el8.x86_64.rpm 13 MB/s | 688 kB 00:00 2023-03-27T10:12:59.655 INFO:teuthology.orchestra.run.smithi159.stdout:(78/131): python3-jinja2-2.10.1-3.el8.noarch.rp 7.9 MB/s | 538 kB 00:00 2023-03-27T10:12:59.672 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2023-03-27T10:12:59.676 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2023-03-27T10:12:59.680 INFO:teuthology.orchestra.run.smithi159.stdout:(79/131): python3-mako-1.0.6-13.el8.noarch.rpm 2.6 MB/s | 157 kB 00:00 2023-03-27T10:12:59.705 INFO:teuthology.orchestra.run.smithi159.stdout:(80/131): oniguruma-6.8.2-2.el8.x86_64.rpm 3.7 MB/s | 187 kB 00:00 2023-03-27T10:12:59.739 INFO:teuthology.orchestra.run.smithi159.stdout:(81/131): automake-1.16.1-7.el8.noarch.rpm 12 MB/s | 713 kB 00:00 2023-03-27T10:12:59.798 INFO:teuthology.orchestra.run.smithi159.stdout:(82/131): python3-jsonpatch-1.21-2.el8.noarch.r 456 kB/s | 27 kB 00:00 2023-03-27T10:12:59.840 INFO:teuthology.orchestra.run.smithi159.stdout:(83/131): flex-2.6.1-9.el8.x86_64.rpm 7.5 MB/s | 320 kB 00:00 2023-03-27T10:12:59.907 INFO:teuthology.orchestra.run.smithi159.stdout:(84/131): python3-babel-2.5.1-7.el8.noarch.rpm 24 MB/s | 4.8 MB 00:00 2023-03-27T10:12:59.932 INFO:teuthology.orchestra.run.smithi159.stdout:(85/131): python3-prettytable-0.7.2-14.el8.noar 476 kB/s | 44 kB 00:00 2023-03-27T10:12:59.974 INFO:teuthology.orchestra.run.smithi159.stdout:(86/131): python3-werkzeug-0.12.2-4.el8.noarch. 11 MB/s | 457 kB 00:00 2023-03-27T10:13:00.041 INFO:teuthology.orchestra.run.smithi159.stdout:(87/131): python3-numpy-f2py-1.14.3-10.el8.x86_ 3.3 MB/s | 225 kB 00:00 2023-03-27T10:13:00.100 INFO:teuthology.orchestra.run.smithi159.stdout:(88/131): python3-numpy-1.14.3-10.el8.x86_64.rp 19 MB/s | 3.7 MB 00:00 2023-03-27T10:13:00.125 INFO:teuthology.orchestra.run.smithi159.stdout:(89/131): uuid-1.6.2-43.el8.x86_64.rpm 764 kB/s | 64 kB 00:00 2023-03-27T10:13:00.150 INFO:teuthology.orchestra.run.smithi159.stdout:(90/131): python3-pyasn1-0.3.7-6.el8.noarch.rpm 2.5 MB/s | 126 kB 00:00 2023-03-27T10:13:00.176 INFO:teuthology.orchestra.run.smithi159.stdout:(91/131): python3-jsonpointer-1.10-11.el8.noarc 397 kB/s | 20 kB 00:00 2023-03-27T10:13:00.201 INFO:teuthology.orchestra.run.smithi159.stdout:(92/131): python3-pyOpenSSL-19.0.0-1.el8.noarch 2.0 MB/s | 103 kB 00:00 2023-03-27T10:13:00.220 INFO:teuthology.orchestra.run.smithi161.stdout:Running transaction check 2023-03-27T10:13:00.255 INFO:teuthology.orchestra.run.smithi159.stdout:(93/131): libtool-2.4.6-25.el8.x86_64.rpm 8.8 MB/s | 709 kB 00:00 2023-03-27T10:13:00.293 INFO:teuthology.orchestra.run.smithi159.stdout:(94/131): libpq-13.5-1.el8.x86_64.rpm 5.1 MB/s | 198 kB 00:00 2023-03-27T10:13:00.339 INFO:teuthology.orchestra.run.smithi161.stdout:Transaction check succeeded. 2023-03-27T10:13:00.339 INFO:teuthology.orchestra.run.smithi161.stdout:Running transaction test 2023-03-27T10:13:00.385 INFO:teuthology.orchestra.run.smithi159.stdout:(95/131): openblas-threads-0.3.15-3.el8.x86_64. 26 MB/s | 4.9 MB 00:00 2023-03-27T10:13:00.510 INFO:teuthology.orchestra.run.smithi159.stdout:(96/131): jq-1.6-3.el8.x86_64.rpm 1.6 MB/s | 202 kB 00:00 2023-03-27T10:13:00.778 INFO:teuthology.orchestra.run.smithi159.stdout:(97/131): openblas-0.3.15-3.el8.x86_64.rpm 9.8 MB/s | 4.7 MB 00:00 2023-03-27T10:13:00.886 INFO:teuthology.orchestra.run.smithi159.stdout:(98/131): librdkafka-0.11.4-3.el8.x86_64.rpm 942 kB/s | 354 kB 00:00 2023-03-27T10:13:00.953 INFO:teuthology.orchestra.run.smithi159.stdout:(99/131): postgresql-contrib-10.21-2.module+el8 12 MB/s | 808 kB 00:00 2023-03-27T10:13:01.070 INFO:teuthology.orchestra.run.smithi159.stdout:(100/131): postgresql-server-10.21-2.module+el8 17 MB/s | 5.1 MB 00:00 2023-03-27T10:13:01.112 INFO:teuthology.orchestra.run.smithi159.stdout:(101/131): postgresql-10.21-2.module+el8.6.0+15 9.5 MB/s | 1.5 MB 00:00 2023-03-27T10:13:01.137 INFO:teuthology.orchestra.run.smithi159.stdout:(102/131): librabbitmq-0.9.0-3.el8.x86_64.rpm 703 kB/s | 47 kB 00:00 2023-03-27T10:13:01.163 INFO:teuthology.orchestra.run.smithi159.stdout:(103/131): libselinux-devel-2.9-5.el8.x86_64.rp 3.9 MB/s | 200 kB 00:00 2023-03-27T10:13:01.188 INFO:teuthology.orchestra.run.smithi159.stdout:(104/131): python3-jwt-1.6.1-2.el8.noarch.rpm 846 kB/s | 43 kB 00:00 2023-03-27T10:13:01.213 INFO:teuthology.orchestra.run.smithi159.stdout:(105/131): pcre2-utf32-10.32-2.el8.x86_64.rpm 4.3 MB/s | 220 kB 00:00 2023-03-27T10:13:01.239 INFO:teuthology.orchestra.run.smithi159.stdout:(106/131): keyutils-libs-devel-1.5.10-9.el8.x86 956 kB/s | 48 kB 00:00 2023-03-27T10:13:01.264 INFO:teuthology.orchestra.run.smithi159.stdout:(107/131): libsepol-devel-2.9-3.el8.x86_64.rpm 1.7 MB/s | 87 kB 00:00 2023-03-27T10:13:01.331 INFO:teuthology.orchestra.run.smithi159.stdout:(108/131): python3-requests-oauthlib-1.0.0-1.el 464 kB/s | 43 kB 00:00 2023-03-27T10:13:01.356 INFO:teuthology.orchestra.run.smithi159.stdout:(109/131): python3-pycparser-2.14-14.el8.noarch 1.2 MB/s | 109 kB 00:00 2023-03-27T10:13:01.381 INFO:teuthology.orchestra.run.smithi159.stdout:(110/131): pcre2-utf16-10.32-2.el8.x86_64.rpm 4.5 MB/s | 229 kB 00:00 2023-03-27T10:13:01.424 INFO:teuthology.orchestra.run.smithi159.stdout:(111/131): krb5-devel-1.18.2-14.el8.x86_64.rpm 8.2 MB/s | 560 kB 00:00 2023-03-27T10:13:01.433 INFO:teuthology.orchestra.run.smithi161.stdout:Transaction test succeeded. 2023-03-27T10:13:01.437 INFO:teuthology.orchestra.run.smithi161.stdout:Running transaction 2023-03-27T10:13:01.441 INFO:teuthology.orchestra.run.smithi159.stdout:(112/131): libaio-devel-0.3.112-1.el8.x86_64.rp 318 kB/s | 19 kB 00:00 2023-03-27T10:13:01.466 INFO:teuthology.orchestra.run.smithi159.stdout:(113/131): python3-oauthlib-2.1.0-1.el8.noarch. 3.6 MB/s | 155 kB 00:00 2023-03-27T10:13:01.499 INFO:teuthology.orchestra.run.smithi159.stdout:(114/131): python3-cryptography-3.2.1-5.el8.x86 9.4 MB/s | 559 kB 00:00 2023-03-27T10:13:01.558 INFO:teuthology.orchestra.run.smithi159.stdout:(115/131): libverto-devel-0.3.0-5.el8.x86_64.rp 194 kB/s | 18 kB 00:00 2023-03-27T10:13:01.592 INFO:teuthology.orchestra.run.smithi159.stdout:(116/131): pcre2-devel-10.32-2.el8.x86_64.rpm 6.4 MB/s | 605 kB 00:00 2023-03-27T10:13:01.617 INFO:teuthology.orchestra.run.smithi159.stdout:(117/131): python3-cffi-1.11.5-5.el8.x86_64.rpm 4.0 MB/s | 238 kB 00:00 2023-03-27T10:13:01.659 INFO:teuthology.orchestra.run.smithi159.stdout:(118/131): xfsprogs-devel-5.0.0-10.el8.x86_64.r 1.7 MB/s | 72 kB 00:00 2023-03-27T10:13:01.710 INFO:teuthology.orchestra.run.smithi159.stdout:(119/131): openssl-devel-1.1.1k-7.el8_6.x86_64. 20 MB/s | 2.3 MB 00:00 2023-03-27T10:13:01.735 INFO:teuthology.orchestra.run.smithi159.stdout:(120/131): fuse-2.9.7-15.el8.x86_64.rpm 1.1 MB/s | 83 kB 00:00 2023-03-27T10:13:01.796 INFO:teuthology.orchestra.run.smithi159.stdout:(121/131): elfutils-libelf-devel-0.186-1.el8.x8 692 kB/s | 60 kB 00:00 2023-03-27T10:13:01.827 INFO:teuthology.orchestra.run.smithi159.stdout:(122/131): libuuid-devel-2.32.1-35.el8.x86_64.r 1.0 MB/s | 98 kB 00:00 2023-03-27T10:13:01.852 INFO:teuthology.orchestra.run.smithi159.stdout:(123/131): zlib-devel-1.2.11-18.el8_5.x86_64.rp 1.0 MB/s | 58 kB 00:00 2023-03-27T10:13:01.878 INFO:teuthology.orchestra.run.smithi159.stdout:(124/131): libcom_err-devel-1.45.6-4.el8.x86_64 770 kB/s | 39 kB 00:00 2023-03-27T10:13:01.920 INFO:teuthology.orchestra.run.smithi159.stdout:(125/131): libgfortran-8.5.0-10.1.el8_6.x86_64. 9.4 MB/s | 644 kB 00:00 2023-03-27T10:13:01.945 INFO:teuthology.orchestra.run.smithi159.stdout:(126/131): libquadmath-8.5.0-10.1.el8_6.x86_64. 2.5 MB/s | 171 kB 00:00 2023-03-27T10:13:01.970 INFO:teuthology.orchestra.run.smithi159.stdout:(127/131): NetworkManager-initscripts-updown-1. 2.7 MB/s | 138 kB 00:00 2023-03-27T10:13:02.604 INFO:teuthology.orchestra.run.smithi159.stdout:(128/131): ceph-test-18.0.0-2533.gb70fdf97.el8. 3.3 MB/s | 45 MB 00:13 2023-03-27T10:13:02.662 INFO:teuthology.orchestra.run.smithi159.stdout:(129/131): iproute-5.15.0-4.el8_6.1.x86_64.rpm 13 MB/s | 798 kB 00:00 2023-03-27T10:13:02.738 INFO:teuthology.orchestra.run.smithi159.stdout:(130/131): librbd1-18.0.0-2533.gb70fdf97.el8.x8 4.6 MB/s | 3.6 MB 00:00 2023-03-27T10:13:02.871 INFO:teuthology.orchestra.run.smithi159.stdout:(131/131): librados2-18.0.0-2533.gb70fdf97.el8. 3.9 MB/s | 3.6 MB 00:00 2023-03-27T10:13:02.874 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2023-03-27T10:13:02.874 INFO:teuthology.orchestra.run.smithi159.stdout:Total 9.2 MB/s | 199 MB 00:21 2023-03-27T10:13:03.693 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2023-03-27T10:13:03.813 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2023-03-27T10:13:03.813 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2023-03-27T10:13:04.593 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2023-03-27T10:13:04.776 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : thrift-0.13.0-2.el8.x86_64 1/134 2023-03-27T10:13:04.907 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2023-03-27T10:13:04.911 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2023-03-27T10:13:05.006 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : liboath-2.6.2-3.el8.x86_64 2/134 2023-03-27T10:13:05.166 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 3/134 2023-03-27T10:13:05.273 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 3/134 2023-03-27T10:13:05.517 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 4/134 2023-03-27T10:13:05.915 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 4/134 2023-03-27T10:13:06.166 INFO:teuthology.orchestra.run.smithi171.stdout: Upgrading : librados2-2:18.0.0-2533.gb70fdf97.el8.x86_64 5/134 2023-03-27T10:13:06.526 INFO:teuthology.orchestra.run.smithi161.stdout: Preparing : 1/1 2023-03-27T10:13:06.560 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librados2-2:18.0.0-2533.gb70fdf97.el8.x86_64 5/134 2023-03-27T10:13:06.729 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : thrift-0.13.0-2.el8.x86_64 1/134 2023-03-27T10:13:06.791 INFO:teuthology.orchestra.run.smithi171.stdout: Upgrading : librbd1-2:18.0.0-2533.gb70fdf97.el8.x86_64 6/134 2023-03-27T10:13:06.903 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : liboath-2.6.2-3.el8.x86_64 2/134 2023-03-27T10:13:06.918 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librbd1-2:18.0.0-2533.gb70fdf97.el8.x86_64 6/134 2023-03-27T10:13:07.114 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 3/134 2023-03-27T10:13:07.132 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libcephfs2-2:18.0.0-2533.gb70fdf97.el8.x86_64 7/134 2023-03-27T10:13:07.202 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 3/134 2023-03-27T10:13:07.263 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: libcephfs2-2:18.0.0-2533.gb70fdf97.el8.x86_64 7/134 2023-03-27T10:13:07.437 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-rados-2:18.0.0-2533.gb70fdf97.el8.x86_64 8/134 2023-03-27T10:13:07.455 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 4/134 2023-03-27T10:13:07.616 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 9/134 2023-03-27T10:13:07.877 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 4/134 2023-03-27T10:13:07.878 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libpq-13.5-1.el8.x86_64 10/134 2023-03-27T10:13:08.077 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 11/134 2023-03-27T10:13:08.096 INFO:teuthology.orchestra.run.smithi161.stdout: Upgrading : librados2-2:18.0.0-2533.gb70fdf97.el8.x86_64 5/134 2023-03-27T10:13:08.462 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 12/134 2023-03-27T10:13:08.481 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: librados2-2:18.0.0-2533.gb70fdf97.el8.x86_64 5/134 2023-03-27T10:13:08.680 INFO:teuthology.orchestra.run.smithi161.stdout: Upgrading : librbd1-2:18.0.0-2533.gb70fdf97.el8.x86_64 6/134 2023-03-27T10:13:08.695 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : postgresql-10.21-2.module+el8.6.0+15342+53518fac 13/134 2023-03-27T10:13:08.808 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: librbd1-2:18.0.0-2533.gb70fdf97.el8.x86_64 6/134 2023-03-27T10:13:08.849 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libcephsqlite-2:18.0.0-2533.gb70fdf97.el8.x86_64 14/134 2023-03-27T10:13:08.990 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: libcephsqlite-2:18.0.0-2533.gb70fdf97.el8.x86_64 14/134 2023-03-27T10:13:09.005 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : libcephfs2-2:18.0.0-2533.gb70fdf97.el8.x86_64 7/134 2023-03-27T10:13:09.104 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: libcephfs2-2:18.0.0-2533.gb70fdf97.el8.x86_64 7/134 2023-03-27T10:13:09.166 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libradosstriper1-2:18.0.0-2533.gb70fdf97.el8.x86 15/134 2023-03-27T10:13:09.293 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-rados-2:18.0.0-2533.gb70fdf97.el8.x86_64 8/134 2023-03-27T10:13:09.317 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: libradosstriper1-2:18.0.0-2533.gb70fdf97.el8.x86 15/134 2023-03-27T10:13:09.483 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 9/134 2023-03-27T10:13:09.569 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 16/134 2023-03-27T10:13:09.751 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : libpq-13.5-1.el8.x86_64 10/134 2023-03-27T10:13:09.975 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 11/134 2023-03-27T10:13:10.026 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 17/134 2023-03-27T10:13:10.210 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 18/134 2023-03-27T10:13:10.250 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2023-03-27T10:13:10.349 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 19/134 2023-03-27T10:13:10.395 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 12/134 2023-03-27T10:13:10.460 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : thrift-0.13.0-2.el8.x86_64 1/134 2023-03-27T10:13:10.493 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 19/134 2023-03-27T10:13:10.684 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : postgresql-10.21-2.module+el8.6.0+15342+53518fac 13/134 2023-03-27T10:13:10.686 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : liboath-2.6.2-3.el8.x86_64 2/134 2023-03-27T10:13:10.690 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 20/134 2023-03-27T10:13:10.935 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 3/134 2023-03-27T10:13:11.104 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 4/134 2023-03-27T10:13:11.113 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : libcephsqlite-2:18.0.0-2533.gb70fdf97.el8.x86_64 14/134 2023-03-27T10:13:11.219 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: libcephsqlite-2:18.0.0-2533.gb70fdf97.el8.x86_64 14/134 2023-03-27T10:13:11.307 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 20/134 2023-03-27T10:13:11.388 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : libradosstriper1-2:18.0.0-2533.gb70fdf97.el8.x86 15/134 2023-03-27T10:13:11.474 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 21/134 2023-03-27T10:13:11.476 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 4/134 2023-03-27T10:13:11.543 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: libradosstriper1-2:18.0.0-2533.gb70fdf97.el8.x86 15/134 2023-03-27T10:13:11.770 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 5/134 2023-03-27T10:13:11.775 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 16/134 2023-03-27T10:13:11.993 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 17/134 2023-03-27T10:13:12.072 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 21/134 2023-03-27T10:13:12.169 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 5/134 2023-03-27T10:13:12.202 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 18/134 2023-03-27T10:13:12.257 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : openblas-0.3.15-3.el8.x86_64 22/134 2023-03-27T10:13:12.346 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 19/134 2023-03-27T10:13:12.403 INFO:teuthology.orchestra.run.smithi159.stdout: Upgrading : librados2-2:18.0.0-2533.gb70fdf97.el8.x86_64 6/134 2023-03-27T10:13:12.511 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 19/134 2023-03-27T10:13:12.697 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 20/134 2023-03-27T10:13:12.796 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: librados2-2:18.0.0-2533.gb70fdf97.el8.x86_64 6/134 2023-03-27T10:13:12.983 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 22/134 2023-03-27T10:13:13.020 INFO:teuthology.orchestra.run.smithi159.stdout: Upgrading : librbd1-2:18.0.0-2533.gb70fdf97.el8.x86_64 7/134 2023-03-27T10:13:13.146 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: librbd1-2:18.0.0-2533.gb70fdf97.el8.x86_64 7/134 2023-03-27T10:13:13.221 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 23/134 2023-03-27T10:13:13.319 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 20/134 2023-03-27T10:13:13.336 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libcephfs2-2:18.0.0-2533.gb70fdf97.el8.x86_64 8/134 2023-03-27T10:13:13.404 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : zlib-devel-1.2.11-18.el8_5.x86_64 24/134 2023-03-27T10:13:13.443 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: libcephfs2-2:18.0.0-2533.gb70fdf97.el8.x86_64 8/134 2023-03-27T10:13:13.488 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 21/134 2023-03-27T10:13:13.649 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-rados-2:18.0.0-2533.gb70fdf97.el8.x86_64 9/134 2023-03-27T10:13:13.650 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 25/134 2023-03-27T10:13:13.804 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-ceph-common-2:18.0.0-2533.gb70fdf97.el8. 26/134 2023-03-27T10:13:13.907 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libpq-13.5-1.el8.x86_64 10/134 2023-03-27T10:13:13.933 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-ceph-argparse-2:18.0.0-2533.gb70fdf97.el 27/134 2023-03-27T10:13:14.126 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 21/134 2023-03-27T10:13:14.156 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 11/134 2023-03-27T10:13:14.161 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cephfs-2:18.0.0-2533.gb70fdf97.el8.x86_6 28/134 2023-03-27T10:13:14.304 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : openblas-0.3.15-3.el8.x86_64 22/134 2023-03-27T10:13:14.357 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 29/134 2023-03-27T10:13:14.557 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 12/134 2023-03-27T10:13:14.815 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : postgresql-10.21-2.module+el8.6.0+15342+53518fac 13/134 2023-03-27T10:13:14.970 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libcephsqlite-2:18.0.0-2533.gb70fdf97.el8.x86_64 14/134 2023-03-27T10:13:15.001 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 22/134 2023-03-27T10:13:15.074 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: libcephsqlite-2:18.0.0-2533.gb70fdf97.el8.x86_64 14/134 2023-03-27T10:13:15.228 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libradosstriper1-2:18.0.0-2533.gb70fdf97.el8.x86 15/134 2023-03-27T10:13:15.269 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 23/134 2023-03-27T10:13:15.308 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: libradosstriper1-2:18.0.0-2533.gb70fdf97.el8.x86 15/134 2023-03-27T10:13:15.453 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 16/134 2023-03-27T10:13:15.461 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : zlib-devel-1.2.11-18.el8_5.x86_64 24/134 2023-03-27T10:13:15.598 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 16/134 2023-03-27T10:13:15.739 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 25/134 2023-03-27T10:13:15.769 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 17/134 2023-03-27T10:13:15.935 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-ceph-common-2:18.0.0-2533.gb70fdf97.el8. 26/134 2023-03-27T10:13:16.064 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-ceph-argparse-2:18.0.0-2533.gb70fdf97.el 27/134 2023-03-27T10:13:16.109 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 29/134 2023-03-27T10:13:16.292 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-cephfs-2:18.0.0-2533.gb70fdf97.el8.x86_6 28/134 2023-03-27T10:13:16.396 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 17/134 2023-03-27T10:13:16.520 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 29/134 2023-03-27T10:13:16.619 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 18/134 2023-03-27T10:13:17.114 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd 30/134 2023-03-27T10:13:17.198 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 18/134 2023-03-27T10:13:17.427 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : openblas-0.3.15-3.el8.x86_64 19/134 2023-03-27T10:13:17.446 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 31/134 2023-03-27T10:13:17.725 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 32/134 2023-03-27T10:13:17.955 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 33/134 2023-03-27T10:13:18.127 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 19/134 2023-03-27T10:13:18.176 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-rsa-4.8-1.el8.noarch 34/134 2023-03-27T10:13:18.276 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 29/134 2023-03-27T10:13:18.366 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 35/134 2023-03-27T10:13:18.409 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 20/134 2023-03-27T10:13:18.562 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-rbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 36/134 2023-03-27T10:13:18.670 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : zlib-devel-1.2.11-18.el8_5.x86_64 21/134 2023-03-27T10:13:18.721 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : librados-devel-2:18.0.0-2533.gb70fdf97.el8.x86_6 37/134 2023-03-27T10:13:18.905 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 38/134 2023-03-27T10:13:18.923 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 22/134 2023-03-27T10:13:19.041 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 39/134 2023-03-27T10:13:19.151 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 23/134 2023-03-27T10:13:19.214 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : uuid-1.6.2-43.el8.x86_64 40/134 2023-03-27T10:13:19.283 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: uuid-1.6.2-43.el8.x86_64 40/134 2023-03-27T10:13:19.334 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd 30/134 2023-03-27T10:13:19.392 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 24/134 2023-03-27T10:13:19.531 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 41/134 2023-03-27T10:13:19.659 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 31/134 2023-03-27T10:13:19.670 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 25/134 2023-03-27T10:13:19.751 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : automake-1.16.1-7.el8.noarch 42/134 2023-03-27T10:13:19.876 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-ceph-common-2:18.0.0-2533.gb70fdf97.el8. 26/134 2023-03-27T10:13:19.947 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 32/134 2023-03-27T10:13:19.947 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 43/134 2023-03-27T10:13:20.029 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 43/134 2023-03-27T10:13:20.046 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-ceph-argparse-2:18.0.0-2533.gb70fdf97.el 27/134 2023-03-27T10:13:20.201 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 33/134 2023-03-27T10:13:20.252 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : jq-1.6-3.el8.x86_64 44/134 2023-03-27T10:13:20.418 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 45/134 2023-03-27T10:13:20.423 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-rsa-4.8-1.el8.noarch 34/134 2023-03-27T10:13:20.613 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libcom_err-devel-1.45.6-4.el8.x86_64 46/134 2023-03-27T10:13:20.614 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 35/134 2023-03-27T10:13:20.757 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libuuid-devel-2.32.1-35.el8.x86_64 47/134 2023-03-27T10:13:20.817 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-rbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 36/134 2023-03-27T10:13:20.916 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : fuse-2.9.7-15.el8.x86_64 48/134 2023-03-27T10:13:20.972 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-cephfs-2:18.0.0-2533.gb70fdf97.el8.x86_6 28/134 2023-03-27T10:13:20.993 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : librados-devel-2:18.0.0-2533.gb70fdf97.el8.x86_6 37/134 2023-03-27T10:13:21.101 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libverto-devel-0.3.0-5.el8.x86_64 49/134 2023-03-27T10:13:21.144 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 38/134 2023-03-27T10:13:21.304 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : pcre2-utf16-10.32-2.el8.x86_64 50/134 2023-03-27T10:13:21.334 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 39/134 2023-03-27T10:13:21.358 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 29/134 2023-03-27T10:13:21.496 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 51/134 2023-03-27T10:13:21.663 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 30/134 2023-03-27T10:13:21.678 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : uuid-1.6.2-43.el8.x86_64 40/134 2023-03-27T10:13:21.890 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 52/134 2023-03-27T10:13:21.921 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 31/134 2023-03-27T10:13:21.928 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: uuid-1.6.2-43.el8.x86_64 40/134 2023-03-27T10:13:22.152 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 32/134 2023-03-27T10:13:22.156 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 41/134 2023-03-27T10:13:22.265 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 53/134 2023-03-27T10:13:22.382 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : automake-1.16.1-7.el8.noarch 42/134 2023-03-27T10:13:22.466 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 54/134 2023-03-27T10:13:22.553 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 43/134 2023-03-27T10:13:22.631 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 43/134 2023-03-27T10:13:22.682 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 55/134 2023-03-27T10:13:22.874 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : jq-1.6-3.el8.x86_64 44/134 2023-03-27T10:13:22.918 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 56/134 2023-03-27T10:13:23.057 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 45/134 2023-03-27T10:13:23.211 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : libcom_err-devel-1.45.6-4.el8.x86_64 46/134 2023-03-27T10:13:23.219 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 57/134 2023-03-27T10:13:23.413 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : libuuid-devel-2.32.1-35.el8.x86_64 47/134 2023-03-27T10:13:23.463 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 58/134 2023-03-27T10:13:23.581 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : fuse-2.9.7-15.el8.x86_64 48/134 2023-03-27T10:13:23.648 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 59/134 2023-03-27T10:13:23.765 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : libverto-devel-0.3.0-5.el8.x86_64 49/134 2023-03-27T10:13:23.867 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 60/134 2023-03-27T10:13:23.915 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 32/134 2023-03-27T10:13:24.003 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : pcre2-utf16-10.32-2.el8.x86_64 50/134 2023-03-27T10:13:24.075 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libsepol-devel-2.9-3.el8.x86_64 61/134 2023-03-27T10:13:24.226 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 51/134 2023-03-27T10:13:24.241 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : keyutils-libs-devel-1.5.10-9.el8.x86_64 62/134 2023-03-27T10:13:24.253 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd 33/134 2023-03-27T10:13:24.480 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-rsa-4.8-1.el8.noarch 34/134 2023-03-27T10:13:24.552 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : pcre2-utf32-10.32-2.el8.x86_64 63/134 2023-03-27T10:13:24.593 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 52/134 2023-03-27T10:13:24.696 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 35/134 2023-03-27T10:13:24.832 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : pcre2-devel-10.32-2.el8.x86_64 64/134 2023-03-27T10:13:24.932 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-rbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 36/134 2023-03-27T10:13:24.954 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 53/134 2023-03-27T10:13:25.076 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libselinux-devel-2.9-5.el8.x86_64 65/134 2023-03-27T10:13:25.147 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : librados-devel-2:18.0.0-2533.gb70fdf97.el8.x86_6 37/134 2023-03-27T10:13:25.197 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 54/134 2023-03-27T10:13:25.325 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libcom_err-devel-1.45.6-4.el8.x86_64 38/134 2023-03-27T10:13:25.339 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : krb5-devel-1.18.2-14.el8.x86_64 66/134 2023-03-27T10:13:25.466 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 55/134 2023-03-27T10:13:25.521 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 67/134 2023-03-27T10:13:25.544 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libuuid-devel-2.32.1-35.el8.x86_64 39/134 2023-03-27T10:13:25.707 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 68/134 2023-03-27T10:13:25.732 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 56/134 2023-03-27T10:13:25.754 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : fuse-2.9.7-15.el8.x86_64 40/134 2023-03-27T10:13:25.955 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libverto-devel-0.3.0-5.el8.x86_64 41/134 2023-03-27T10:13:25.992 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 69/134 2023-03-27T10:13:26.069 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 57/134 2023-03-27T10:13:26.203 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-waitress-1.2.1-2.el8.1.noarch 70/134 2023-03-27T10:13:26.209 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : pcre2-utf16-10.32-2.el8.x86_64 42/134 2023-03-27T10:13:26.319 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 58/134 2023-03-27T10:13:26.384 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 71/134 2023-03-27T10:13:26.433 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 43/134 2023-03-27T10:13:26.548 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 72/134 2023-03-27T10:13:26.557 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 59/134 2023-03-27T10:13:26.738 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 73/134 2023-03-27T10:13:26.765 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 60/134 2023-03-27T10:13:26.833 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 44/134 2023-03-27T10:13:26.966 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : libsepol-devel-2.9-3.el8.x86_64 61/134 2023-03-27T10:13:26.983 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 74/134 2023-03-27T10:13:27.148 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 75/134 2023-03-27T10:13:27.171 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 45/134 2023-03-27T10:13:27.173 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : keyutils-libs-devel-1.5.10-9.el8.x86_64 62/134 2023-03-27T10:13:27.307 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 76/134 2023-03-27T10:13:27.403 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 46/134 2023-03-27T10:13:27.452 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 77/134 2023-03-27T10:13:27.534 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : pcre2-utf32-10.32-2.el8.x86_64 63/134 2023-03-27T10:13:27.656 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 47/134 2023-03-27T10:13:27.727 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 78/134 2023-03-27T10:13:27.815 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : pcre2-devel-10.32-2.el8.x86_64 64/134 2023-03-27T10:13:27.940 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 48/134 2023-03-27T10:13:27.959 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 79/134 2023-03-27T10:13:28.082 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : libselinux-devel-2.9-5.el8.x86_64 65/134 2023-03-27T10:13:28.185 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 80/134 2023-03-27T10:13:28.320 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 49/134 2023-03-27T10:13:28.337 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : krb5-devel-1.18.2-14.el8.x86_64 66/134 2023-03-27T10:13:28.419 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-portend-2.6-1.el8.noarch 81/134 2023-03-27T10:13:28.493 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 67/134 2023-03-27T10:13:28.584 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 50/134 2023-03-27T10:13:28.637 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 82/134 2023-03-27T10:13:28.689 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 68/134 2023-03-27T10:13:28.819 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 51/134 2023-03-27T10:13:28.836 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 83/134 2023-03-27T10:13:28.969 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 69/134 2023-03-27T10:13:29.030 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 52/134 2023-03-27T10:13:29.045 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 84/134 2023-03-27T10:13:29.200 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-waitress-1.2.1-2.el8.1.noarch 70/134 2023-03-27T10:13:29.230 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libsepol-devel-2.9-3.el8.x86_64 53/134 2023-03-27T10:13:29.340 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 71/134 2023-03-27T10:13:29.446 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : keyutils-libs-devel-1.5.10-9.el8.x86_64 54/134 2023-03-27T10:13:29.505 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 72/134 2023-03-27T10:13:29.738 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 73/134 2023-03-27T10:13:29.760 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : pcre2-utf32-10.32-2.el8.x86_64 55/134 2023-03-27T10:13:29.948 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 74/134 2023-03-27T10:13:30.039 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : pcre2-devel-10.32-2.el8.x86_64 56/134 2023-03-27T10:13:30.143 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 85/134 2023-03-27T10:13:30.147 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 75/134 2023-03-27T10:13:30.280 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libselinux-devel-2.9-5.el8.x86_64 57/134 2023-03-27T10:13:30.338 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 76/134 2023-03-27T10:13:30.458 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 86/134 2023-03-27T10:13:30.500 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 77/134 2023-03-27T10:13:30.541 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : krb5-devel-1.18.2-14.el8.x86_64 58/134 2023-03-27T10:13:30.733 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 59/134 2023-03-27T10:13:30.759 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 78/134 2023-03-27T10:13:30.762 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 87/134 2023-03-27T10:13:30.902 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 60/134 2023-03-27T10:13:30.997 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 79/134 2023-03-27T10:13:31.109 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : uuid-1.6.2-43.el8.x86_64 61/134 2023-03-27T10:13:31.122 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 88/134 2023-03-27T10:13:31.182 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: uuid-1.6.2-43.el8.x86_64 61/134 2023-03-27T10:13:31.208 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 80/134 2023-03-27T10:13:31.379 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 89/134 2023-03-27T10:13:31.449 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-portend-2.6-1.el8.noarch 81/134 2023-03-27T10:13:31.480 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 62/134 2023-03-27T10:13:31.622 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 90/134 2023-03-27T10:13:31.659 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 82/134 2023-03-27T10:13:31.703 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : automake-1.16.1-7.el8.noarch 63/134 2023-03-27T10:13:31.850 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 83/134 2023-03-27T10:13:31.875 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 64/134 2023-03-27T10:13:31.952 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 64/134 2023-03-27T10:13:32.102 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 84/134 2023-03-27T10:13:32.196 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : jq-1.6-3.el8.x86_64 65/134 2023-03-27T10:13:32.386 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 91/134 2023-03-27T10:13:32.409 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 66/134 2023-03-27T10:13:32.545 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : librgw2-2:18.0.0-2533.gb70fdf97.el8.x86_64 92/134 2023-03-27T10:13:32.583 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 67/134 2023-03-27T10:13:32.619 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librgw2-2:18.0.0-2533.gb70fdf97.el8.x86_64 92/134 2023-03-27T10:13:32.753 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 68/134 2023-03-27T10:13:32.837 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-rgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 93/134 2023-03-27T10:13:33.030 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 69/134 2023-03-27T10:13:33.290 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-waitress-1.2.1-2.el8.1.noarch 70/134 2023-03-27T10:13:33.295 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 85/134 2023-03-27T10:13:33.438 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 71/134 2023-03-27T10:13:33.613 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 86/134 2023-03-27T10:13:33.636 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 72/134 2023-03-27T10:13:33.861 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 73/134 2023-03-27T10:13:33.901 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 87/134 2023-03-27T10:13:34.105 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 74/134 2023-03-27T10:13:34.252 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 88/134 2023-03-27T10:13:34.321 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 75/134 2023-03-27T10:13:34.520 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 76/134 2023-03-27T10:13:34.550 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 89/134 2023-03-27T10:13:34.673 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 77/134 2023-03-27T10:13:34.751 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 90/134 2023-03-27T10:13:34.925 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 78/134 2023-03-27T10:13:35.154 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 79/134 2023-03-27T10:13:35.381 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 80/134 2023-03-27T10:13:35.536 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 91/134 2023-03-27T10:13:35.647 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-portend-2.6-1.el8.noarch 81/134 2023-03-27T10:13:35.726 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : librgw2-2:18.0.0-2533.gb70fdf97.el8.x86_64 92/134 2023-03-27T10:13:35.797 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: librgw2-2:18.0.0-2533.gb70fdf97.el8.x86_64 92/134 2023-03-27T10:13:35.891 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 82/134 2023-03-27T10:13:36.010 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : python3-rgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 93/134 2023-03-27T10:13:36.091 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 83/134 2023-03-27T10:13:36.342 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 84/134 2023-03-27T10:13:36.419 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 94/134 2023-03-27T10:13:36.653 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 94/134 2023-03-27T10:13:37.197 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 94/134 2023-03-27T10:13:37.455 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 85/134 2023-03-27T10:13:37.501 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-base-2:18.0.0-2533.gb70fdf97.el8.x86_64 95/134 2023-03-27T10:13:37.590 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-base-2:18.0.0-2533.gb70fdf97.el8.x86_64 95/134 2023-03-27T10:13:37.736 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-selinux-2:18.0.0-2533.gb70fdf97.el8.x86_64 96/134 2023-03-27T10:13:37.804 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 86/134 2023-03-27T10:13:38.100 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 87/134 2023-03-27T10:13:38.459 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 88/134 2023-03-27T10:13:38.784 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 89/134 2023-03-27T10:13:38.998 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 90/134 2023-03-27T10:13:39.675 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 94/134 2023-03-27T10:13:39.818 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 91/134 2023-03-27T10:13:39.901 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 94/134 2023-03-27T10:13:39.991 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : librgw2-2:18.0.0-2533.gb70fdf97.el8.x86_64 92/134 2023-03-27T10:13:40.065 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: librgw2-2:18.0.0-2533.gb70fdf97.el8.x86_64 92/134 2023-03-27T10:13:40.275 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-rgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 93/134 2023-03-27T10:13:40.461 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 94/134 2023-03-27T10:13:40.771 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-base-2:18.0.0-2533.gb70fdf97.el8.x86_64 95/134 2023-03-27T10:13:40.891 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: ceph-base-2:18.0.0-2533.gb70fdf97.el8.x86_64 95/134 2023-03-27T10:13:41.034 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-selinux-2:18.0.0-2533.gb70fdf97.el8.x86_64 96/134 2023-03-27T10:13:43.963 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 94/134 2023-03-27T10:13:44.207 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 94/134 2023-03-27T10:13:44.759 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 94/134 2023-03-27T10:13:45.086 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-base-2:18.0.0-2533.gb70fdf97.el8.x86_64 95/134 2023-03-27T10:13:45.158 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-base-2:18.0.0-2533.gb70fdf97.el8.x86_64 95/134 2023-03-27T10:13:45.349 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-selinux-2:18.0.0-2533.gb70fdf97.el8.x86_64 96/134 2023-03-27T10:13:56.701 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-selinux-2:18.0.0-2533.gb70fdf97.el8.x86_64 96/134 2023-03-27T10:13:56.701 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /sys 2023-03-27T10:13:56.701 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /proc 2023-03-27T10:13:56.701 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /dev 2023-03-27T10:13:56.702 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /run 2023-03-27T10:13:56.702 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /mnt 2023-03-27T10:13:56.702 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /var/tmp 2023-03-27T10:13:56.702 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /home 2023-03-27T10:13:56.702 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /tmp 2023-03-27T10:13:56.702 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /dev 2023-03-27T10:13:56.702 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:13:57.058 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-volume-2:18.0.0-2533.gb70fdf97.el8.noarch 97/134 2023-03-27T10:13:58.506 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-volume-2:18.0.0-2533.gb70fdf97.el8.noarch 97/134 2023-03-27T10:13:58.672 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-osd-2:18.0.0-2533.gb70fdf97.el8.x86_64 98/134 2023-03-27T10:13:58.889 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-osd-2:18.0.0-2533.gb70fdf97.el8.x86_64 98/134 2023-03-27T10:13:59.116 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mds-2:18.0.0-2533.gb70fdf97.el8.x86_64 99/134 2023-03-27T10:13:59.495 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mds-2:18.0.0-2533.gb70fdf97.el8.x86_64 99/134 2023-03-27T10:13:59.637 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: ceph-selinux-2:18.0.0-2533.gb70fdf97.el8.x86_64 96/134 2023-03-27T10:13:59.637 INFO:teuthology.orchestra.run.smithi161.stdout:skipping the directory /sys 2023-03-27T10:13:59.637 INFO:teuthology.orchestra.run.smithi161.stdout:skipping the directory /proc 2023-03-27T10:13:59.637 INFO:teuthology.orchestra.run.smithi161.stdout:skipping the directory /dev 2023-03-27T10:13:59.637 INFO:teuthology.orchestra.run.smithi161.stdout:skipping the directory /run 2023-03-27T10:13:59.637 INFO:teuthology.orchestra.run.smithi161.stdout:skipping the directory /mnt 2023-03-27T10:13:59.638 INFO:teuthology.orchestra.run.smithi161.stdout:skipping the directory /var/tmp 2023-03-27T10:13:59.638 INFO:teuthology.orchestra.run.smithi161.stdout:skipping the directory /home 2023-03-27T10:13:59.638 INFO:teuthology.orchestra.run.smithi161.stdout:skipping the directory /tmp 2023-03-27T10:13:59.638 INFO:teuthology.orchestra.run.smithi161.stdout:skipping the directory /dev 2023-03-27T10:13:59.638 INFO:teuthology.orchestra.run.smithi161.stdout: 2023-03-27T10:13:59.666 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mon-2:18.0.0-2533.gb70fdf97.el8.x86_64 100/134 2023-03-27T10:13:59.684 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mon-2:18.0.0-2533.gb70fdf97.el8.x86_64 100/134 2023-03-27T10:14:00.006 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-volume-2:18.0.0-2533.gb70fdf97.el8.noarch 97/134 2023-03-27T10:14:01.314 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: cephadm-2:18.0.0-2533.gb70fdf97.el8.noarch 101/134 2023-03-27T10:14:01.473 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: ceph-volume-2:18.0.0-2533.gb70fdf97.el8.noarch 97/134 2023-03-27T10:14:01.491 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : cephadm-2:18.0.0-2533.gb70fdf97.el8.noarch 101/134 2023-03-27T10:14:01.613 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-prometheus-alerts-2:18.0.0-2533.gb70fdf97.e 102/134 2023-03-27T10:14:01.652 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-osd-2:18.0.0-2533.gb70fdf97.el8.x86_64 98/134 2023-03-27T10:14:01.819 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-grafana-dashboards-2:18.0.0-2533.gb70fdf97. 103/134 2023-03-27T10:14:01.870 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: ceph-osd-2:18.0.0-2533.gb70fdf97.el8.x86_64 98/134 2023-03-27T10:14:02.049 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-cephadm-2:18.0.0-2533.gb70fdf97.el8.noa 104/134 2023-03-27T10:14:02.097 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-mds-2:18.0.0-2533.gb70fdf97.el8.x86_64 99/134 2023-03-27T10:14:02.264 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.0.0-2533.gb70fdf97.el8.noa 104/134 2023-03-27T10:14:02.477 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: ceph-mds-2:18.0.0-2533.gb70fdf97.el8.x86_64 99/134 2023-03-27T10:14:02.580 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-dashboard-2:18.0.0-2533.gb70fdf97.el8.n 105/134 2023-03-27T10:14:02.655 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-mon-2:18.0.0-2533.gb70fdf97.el8.x86_64 100/134 2023-03-27T10:14:02.673 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: ceph-mon-2:18.0.0-2533.gb70fdf97.el8.x86_64 100/134 2023-03-27T10:14:03.538 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.0.0-2533.gb70fdf97.el8.n 105/134 2023-03-27T10:14:03.732 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-diskprediction-local-2:18.0.0-2533.gb70 106/134 2023-03-27T10:14:03.822 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.0.0-2533.gb70 106/134 2023-03-27T10:14:04.173 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-modules-core-2:18.0.0-2533.gb70fdf97.el 107/134 2023-03-27T10:14:04.182 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-selinux-2:18.0.0-2533.gb70fdf97.el8.x86_64 96/134 2023-03-27T10:14:04.182 INFO:teuthology.orchestra.run.smithi159.stdout:skipping the directory /sys 2023-03-27T10:14:04.182 INFO:teuthology.orchestra.run.smithi159.stdout:skipping the directory /proc 2023-03-27T10:14:04.182 INFO:teuthology.orchestra.run.smithi159.stdout:skipping the directory /dev 2023-03-27T10:14:04.182 INFO:teuthology.orchestra.run.smithi159.stdout:skipping the directory /run 2023-03-27T10:14:04.183 INFO:teuthology.orchestra.run.smithi159.stdout:skipping the directory /mnt 2023-03-27T10:14:04.183 INFO:teuthology.orchestra.run.smithi159.stdout:skipping the directory /var/tmp 2023-03-27T10:14:04.183 INFO:teuthology.orchestra.run.smithi159.stdout:skipping the directory /home 2023-03-27T10:14:04.183 INFO:teuthology.orchestra.run.smithi159.stdout:skipping the directory /tmp 2023-03-27T10:14:04.183 INFO:teuthology.orchestra.run.smithi159.stdout:skipping the directory /dev 2023-03-27T10:14:04.183 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:14:04.363 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: cephadm-2:18.0.0-2533.gb70fdf97.el8.noarch 101/134 2023-03-27T10:14:04.398 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-2:18.0.0-2533.gb70fdf97.el8.x86_64 108/134 2023-03-27T10:14:04.434 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mgr-2:18.0.0-2533.gb70fdf97.el8.x86_64 108/134 2023-03-27T10:14:04.562 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : cephadm-2:18.0.0-2533.gb70fdf97.el8.noarch 101/134 2023-03-27T10:14:04.571 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-volume-2:18.0.0-2533.gb70fdf97.el8.noarch 97/134 2023-03-27T10:14:04.590 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-rook-2:18.0.0-2533.gb70fdf97.el8.noarch 109/134 2023-03-27T10:14:04.602 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mgr-rook-2:18.0.0-2533.gb70fdf97.el8.noarch 109/134 2023-03-27T10:14:04.728 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-prometheus-alerts-2:18.0.0-2533.gb70fdf97.e 102/134 2023-03-27T10:14:04.909 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-grafana-dashboards-2:18.0.0-2533.gb70fdf97. 103/134 2023-03-27T10:14:05.096 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-mgr-cephadm-2:18.0.0-2533.gb70fdf97.el8.noa 104/134 2023-03-27T10:14:05.313 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.0.0-2533.gb70fdf97.el8.noa 104/134 2023-03-27T10:14:05.584 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-mgr-dashboard-2:18.0.0-2533.gb70fdf97.el8.n 105/134 2023-03-27T10:14:05.843 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-2:18.0.0-2533.gb70fdf97.el8.x86_64 110/134 2023-03-27T10:14:06.021 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-volume-2:18.0.0-2533.gb70fdf97.el8.noarch 97/134 2023-03-27T10:14:06.071 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-radosgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 111/134 2023-03-27T10:14:06.104 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-radosgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 111/134 2023-03-27T10:14:06.236 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-osd-2:18.0.0-2533.gb70fdf97.el8.x86_64 98/134 2023-03-27T10:14:06.265 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-immutable-object-cache-2:18.0.0-2533.gb70fd 112/134 2023-03-27T10:14:06.306 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.0.0-2533.gb70fd 112/134 2023-03-27T10:14:06.454 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-osd-2:18.0.0-2533.gb70fdf97.el8.x86_64 98/134 2023-03-27T10:14:06.456 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : cephfs-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 113/134 2023-03-27T10:14:06.548 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.0.0-2533.gb70fdf97.el8.n 105/134 2023-03-27T10:14:06.671 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-mds-2:18.0.0-2533.gb70fdf97.el8.x86_64 99/134 2023-03-27T10:14:06.754 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-mgr-diskprediction-local-2:18.0.0-2533.gb70 106/134 2023-03-27T10:14:06.801 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: cephfs-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 113/134 2023-03-27T10:14:06.846 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.0.0-2533.gb70 106/134 2023-03-27T10:14:06.973 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : rbd-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 114/134 2023-03-27T10:14:07.051 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-mds-2:18.0.0-2533.gb70fdf97.el8.x86_64 99/134 2023-03-27T10:14:07.220 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-mgr-modules-core-2:18.0.0-2533.gb70fdf97.el 107/134 2023-03-27T10:14:07.220 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-mon-2:18.0.0-2533.gb70fdf97.el8.x86_64 100/134 2023-03-27T10:14:07.237 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-mon-2:18.0.0-2533.gb70fdf97.el8.x86_64 100/134 2023-03-27T10:14:07.417 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-mgr-2:18.0.0-2533.gb70fdf97.el8.x86_64 108/134 2023-03-27T10:14:07.452 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: ceph-mgr-2:18.0.0-2533.gb70fdf97.el8.x86_64 108/134 2023-03-27T10:14:07.654 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-mgr-rook-2:18.0.0-2533.gb70fdf97.el8.noarch 109/134 2023-03-27T10:14:07.665 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: ceph-mgr-rook-2:18.0.0-2533.gb70fdf97.el8.noarch 109/134 2023-03-27T10:14:08.972 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-2:18.0.0-2533.gb70fdf97.el8.x86_64 110/134 2023-03-27T10:14:09.037 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: cephadm-2:18.0.0-2533.gb70fdf97.el8.noarch 101/134 2023-03-27T10:14:09.151 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-radosgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 111/134 2023-03-27T10:14:09.185 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: ceph-radosgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 111/134 2023-03-27T10:14:09.255 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : cephadm-2:18.0.0-2533.gb70fdf97.el8.noarch 101/134 2023-03-27T10:14:09.354 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-immutable-object-cache-2:18.0.0-2533.gb70fd 112/134 2023-03-27T10:14:09.396 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.0.0-2533.gb70fd 112/134 2023-03-27T10:14:09.409 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-prometheus-alerts-2:18.0.0-2533.gb70fdf97.e 102/134 2023-03-27T10:14:09.588 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : cephfs-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 113/134 2023-03-27T10:14:09.632 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-grafana-dashboards-2:18.0.0-2533.gb70fdf97. 103/134 2023-03-27T10:14:09.845 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-mgr-cephadm-2:18.0.0-2533.gb70fdf97.el8.noa 104/134 2023-03-27T10:14:09.933 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: cephfs-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 113/134 2023-03-27T10:14:10.056 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.0.0-2533.gb70fdf97.el8.noa 104/134 2023-03-27T10:14:10.113 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : rbd-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 114/134 2023-03-27T10:14:10.235 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-mgr-dashboard-2:18.0.0-2533.gb70fdf97.el8.n 105/134 2023-03-27T10:14:11.194 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.0.0-2533.gb70fdf97.el8.n 105/134 2023-03-27T10:14:11.226 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: rbd-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 114/134 2023-03-27T10:14:11.394 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-mgr-diskprediction-local-2:18.0.0-2533.gb70 106/134 2023-03-27T10:14:11.484 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.0.0-2533.gb70 106/134 2023-03-27T10:14:11.869 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-mgr-modules-core-2:18.0.0-2533.gb70fdf97.el 107/134 2023-03-27T10:14:12.101 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-test-2:18.0.0-2533.gb70fdf97.el8.x86_64 115/134 2023-03-27T10:14:12.120 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-mgr-2:18.0.0-2533.gb70fdf97.el8.x86_64 108/134 2023-03-27T10:14:12.156 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-mgr-2:18.0.0-2533.gb70fdf97.el8.x86_64 108/134 2023-03-27T10:14:12.344 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-mgr-rook-2:18.0.0-2533.gb70fdf97.el8.noarch 109/134 2023-03-27T10:14:12.356 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-mgr-rook-2:18.0.0-2533.gb70fdf97.el8.noarch 109/134 2023-03-27T10:14:12.436 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : openssl-devel-1:1.1.1k-7.el8_6.x86_64 116/134 2023-03-27T10:14:12.616 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 117/134 2023-03-27T10:14:12.838 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : xfsprogs-devel-5.0.0-10.el8.x86_64 118/134 2023-03-27T10:14:13.030 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libtool-2.4.6-25.el8.x86_64 119/134 2023-03-27T10:14:13.223 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: libtool-2.4.6-25.el8.x86_64 119/134 2023-03-27T10:14:13.485 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : postgresql-contrib-10.21-2.module+el8.6.0+15342+ 120/134 2023-03-27T10:14:13.659 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libcephfs-devel-2:18.0.0-2533.gb70fdf97.el8.x86_ 121/134 2023-03-27T10:14:13.661 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-2:18.0.0-2533.gb70fdf97.el8.x86_64 110/134 2023-03-27T10:14:13.815 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : elfutils-libelf-devel-0.186-1.el8.x86_64 122/134 2023-03-27T10:14:13.894 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-radosgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 111/134 2023-03-27T10:14:13.927 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-radosgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 111/134 2023-03-27T10:14:14.136 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-immutable-object-cache-2:18.0.0-2533.gb70fd 112/134 2023-03-27T10:14:14.178 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.0.0-2533.gb70fd 112/134 2023-03-27T10:14:14.369 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : cephfs-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 113/134 2023-03-27T10:14:14.401 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: rbd-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 114/134 2023-03-27T10:14:14.714 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: cephfs-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 113/134 2023-03-27T10:14:14.869 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : rbd-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 114/134 2023-03-27T10:14:15.305 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-test-2:18.0.0-2533.gb70fdf97.el8.x86_64 115/134 2023-03-27T10:14:15.617 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : openssl-devel-1:1.1.1k-7.el8_6.x86_64 116/134 2023-03-27T10:14:15.831 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : ceph-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 117/134 2023-03-27T10:14:15.954 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-03-27T10:14:16.124 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : xfsprogs-devel-5.0.0-10.el8.x86_64 118/134 2023-03-27T10:14:16.222 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-03-27T10:14:16.240 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-03-27T10:14:16.303 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : libtool-2.4.6-25.el8.x86_64 119/134 2023-03-27T10:14:16.440 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : cephfs-top-2:18.0.0-2533.gb70fdf97.el8.noarch 124/134 2023-03-27T10:14:16.494 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: libtool-2.4.6-25.el8.x86_64 119/134 2023-03-27T10:14:16.721 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : rbd-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 125/134 2023-03-27T10:14:16.757 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : postgresql-contrib-10.21-2.module+el8.6.0+15342+ 120/134 2023-03-27T10:14:16.965 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : libcephfs-devel-2:18.0.0-2533.gb70fdf97.el8.x86_ 121/134 2023-03-27T10:14:17.015 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : rbd-nbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 126/134 2023-03-27T10:14:17.087 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : elfutils-libelf-devel-0.186-1.el8.x86_64 122/134 2023-03-27T10:14:17.432 INFO:teuthology.orchestra.run.smithi171.stdout: Upgrading : iproute-5.15.0-4.el8_6.1.x86_64 127/134 2023-03-27T10:14:17.646 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : flex-2.6.1-9.el8.x86_64 128/134 2023-03-27T10:14:17.728 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: flex-2.6.1-9.el8.x86_64 128/134 2023-03-27T10:14:17.938 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : bison-3.0.4-10.el8.x86_64 129/134 2023-03-27T10:14:17.950 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: bison-3.0.4-10.el8.x86_64 129/134 2023-03-27T10:14:18.113 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : NetworkManager-initscripts-updown-1:1.36.0-9.el8 130/134 2023-03-27T10:14:18.120 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: NetworkManager-initscripts-updown-1:1.36.0-9.el8 130/134 2023-03-27T10:14:18.282 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libaio-devel-0.3.112-1.el8.x86_64 131/134 2023-03-27T10:14:18.282 INFO:teuthology.orchestra.run.smithi171.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-03-27T10:14:18.499 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-03-27T10:14:18.500 INFO:teuthology.orchestra.run.smithi171.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 133/134 2023-03-27T10:14:18.771 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 133/134 2023-03-27T10:14:18.978 INFO:teuthology.orchestra.run.smithi171.stdout: Cleanup : iproute-5.15.0-4.el8.x86_64 134/134 2023-03-27T10:14:19.156 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: rbd-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 114/134 2023-03-27T10:14:19.269 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: iproute-5.15.0-4.el8.x86_64 134/134 2023-03-27T10:14:19.269 INFO:teuthology.orchestra.run.smithi171.stdout:[/usr/lib/tmpfiles.d/journal-nocow.conf:26] Failed to resolve specifier: uninitialized /etc detected, skipping 2023-03-27T10:14:19.269 INFO:teuthology.orchestra.run.smithi171.stdout:All rules containing unresolvable specifiers will be skipped. 2023-03-27T10:14:19.269 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:14:19.282 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-03-27T10:14:19.482 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-test-2:18.0.0-2533.gb70fdf97.el8.x86_64 115/134 2023-03-27T10:14:19.497 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-2:18.0.0-2533.gb70fdf97.el8.x86_64 1/134 2023-03-27T10:14:19.497 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-base-2:18.0.0-2533.gb70fdf97.el8.x86_64 2/134 2023-03-27T10:14:19.498 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 3/134 2023-03-27T10:14:19.498 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 4/134 2023-03-27T10:14:19.498 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-immutable-object-cache-2:18.0.0-2533.gb70fd 5/134 2023-03-27T10:14:19.498 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mds-2:18.0.0-2533.gb70fdf97.el8.x86_64 6/134 2023-03-27T10:14:19.498 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-2:18.0.0-2533.gb70fdf97.el8.x86_64 7/134 2023-03-27T10:14:19.498 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mon-2:18.0.0-2533.gb70fdf97.el8.x86_64 8/134 2023-03-27T10:14:19.498 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-osd-2:18.0.0-2533.gb70fdf97.el8.x86_64 9/134 2023-03-27T10:14:19.498 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-radosgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 10/134 2023-03-27T10:14:19.499 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-selinux-2:18.0.0-2533.gb70fdf97.el8.x86_64 11/134 2023-03-27T10:14:19.499 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-test-2:18.0.0-2533.gb70fdf97.el8.x86_64 12/134 2023-03-27T10:14:19.499 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : cephfs-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 13/134 2023-03-27T10:14:19.499 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libcephfs-devel-2:18.0.0-2533.gb70fdf97.el8.x86_ 14/134 2023-03-27T10:14:19.499 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libcephfs2-2:18.0.0-2533.gb70fdf97.el8.x86_64 15/134 2023-03-27T10:14:19.499 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libcephsqlite-2:18.0.0-2533.gb70fdf97.el8.x86_64 16/134 2023-03-27T10:14:19.499 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librados-devel-2:18.0.0-2533.gb70fdf97.el8.x86_6 17/134 2023-03-27T10:14:19.500 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libradosstriper1-2:18.0.0-2533.gb70fdf97.el8.x86 18/134 2023-03-27T10:14:19.500 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librgw2-2:18.0.0-2533.gb70fdf97.el8.x86_64 19/134 2023-03-27T10:14:19.501 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-ceph-argparse-2:18.0.0-2533.gb70fdf97.el 20/134 2023-03-27T10:14:19.501 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-ceph-common-2:18.0.0-2533.gb70fdf97.el8. 21/134 2023-03-27T10:14:19.501 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cephfs-2:18.0.0-2533.gb70fdf97.el8.x86_6 22/134 2023-03-27T10:14:19.501 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-rados-2:18.0.0-2533.gb70fdf97.el8.x86_64 23/134 2023-03-27T10:14:19.501 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-rbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 24/134 2023-03-27T10:14:19.501 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-rgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 25/134 2023-03-27T10:14:19.501 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : rbd-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 26/134 2023-03-27T10:14:19.502 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : rbd-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 27/134 2023-03-27T10:14:19.508 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : rbd-nbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 28/134 2023-03-27T10:14:19.508 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-grafana-dashboards-2:18.0.0-2533.gb70fdf97. 29/134 2023-03-27T10:14:19.509 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-cephadm-2:18.0.0-2533.gb70fdf97.el8.noa 30/134 2023-03-27T10:14:19.509 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-dashboard-2:18.0.0-2533.gb70fdf97.el8.n 31/134 2023-03-27T10:14:19.509 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.0.0-2533.gb70 32/134 2023-03-27T10:14:19.509 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-modules-core-2:18.0.0-2533.gb70fdf97.el 33/134 2023-03-27T10:14:19.509 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-rook-2:18.0.0-2533.gb70fdf97.el8.noarch 34/134 2023-03-27T10:14:19.509 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-prometheus-alerts-2:18.0.0-2533.gb70fdf97.e 35/134 2023-03-27T10:14:19.509 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-volume-2:18.0.0-2533.gb70fdf97.el8.noarch 36/134 2023-03-27T10:14:19.509 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : cephadm-2:18.0.0-2533.gb70fdf97.el8.noarch 37/134 2023-03-27T10:14:19.510 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : cephfs-top-2:18.0.0-2533.gb70fdf97.el8.noarch 38/134 2023-03-27T10:14:19.510 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 39/134 2023-03-27T10:14:19.510 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 40/134 2023-03-27T10:14:19.510 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 41/134 2023-03-27T10:14:19.510 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 42/134 2023-03-27T10:14:19.510 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 43/134 2023-03-27T10:14:19.510 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 44/134 2023-03-27T10:14:19.510 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 45/134 2023-03-27T10:14:19.511 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 46/134 2023-03-27T10:14:19.511 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 47/134 2023-03-27T10:14:19.511 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 48/134 2023-03-27T10:14:19.511 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 49/134 2023-03-27T10:14:19.511 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 50/134 2023-03-27T10:14:19.511 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 51/134 2023-03-27T10:14:19.511 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 52/134 2023-03-27T10:14:19.512 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 53/134 2023-03-27T10:14:19.512 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 54/134 2023-03-27T10:14:19.512 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 55/134 2023-03-27T10:14:19.512 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 56/134 2023-03-27T10:14:19.512 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-portend-2.6-1.el8.noarch 57/134 2023-03-27T10:14:19.512 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 58/134 2023-03-27T10:14:19.512 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 59/134 2023-03-27T10:14:19.512 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-rsa-4.8-1.el8.noarch 60/134 2023-03-27T10:14:19.513 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 61/134 2023-03-27T10:14:19.513 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 62/134 2023-03-27T10:14:19.513 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 63/134 2023-03-27T10:14:19.513 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 64/134 2023-03-27T10:14:19.513 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-waitress-1.2.1-2.el8.1.noarch 65/134 2023-03-27T10:14:19.513 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 66/134 2023-03-27T10:14:19.513 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 67/134 2023-03-27T10:14:19.513 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 68/134 2023-03-27T10:14:19.514 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 69/134 2023-03-27T10:14:19.514 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 70/134 2023-03-27T10:14:19.514 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 71/134 2023-03-27T10:14:19.514 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 72/134 2023-03-27T10:14:19.514 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libselinux-devel-2.9-5.el8.x86_64 73/134 2023-03-27T10:14:19.514 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 74/134 2023-03-27T10:14:19.514 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : pcre2-utf32-10.32-2.el8.x86_64 75/134 2023-03-27T10:14:19.514 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : keyutils-libs-devel-1.5.10-9.el8.x86_64 76/134 2023-03-27T10:14:19.515 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libsepol-devel-2.9-3.el8.x86_64 77/134 2023-03-27T10:14:19.515 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 78/134 2023-03-27T10:14:19.515 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 79/134 2023-03-27T10:14:19.515 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : pcre2-utf16-10.32-2.el8.x86_64 80/134 2023-03-27T10:14:19.515 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : krb5-devel-1.18.2-14.el8.x86_64 81/134 2023-03-27T10:14:19.515 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libaio-devel-0.3.112-1.el8.x86_64 82/134 2023-03-27T10:14:19.515 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 83/134 2023-03-27T10:14:19.515 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 84/134 2023-03-27T10:14:19.516 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libverto-devel-0.3.0-5.el8.x86_64 85/134 2023-03-27T10:14:19.516 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : pcre2-devel-10.32-2.el8.x86_64 86/134 2023-03-27T10:14:19.516 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 87/134 2023-03-27T10:14:19.516 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : openssl-devel-1:1.1.1k-7.el8_6.x86_64 88/134 2023-03-27T10:14:19.516 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : xfsprogs-devel-5.0.0-10.el8.x86_64 89/134 2023-03-27T10:14:19.516 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 90/134 2023-03-27T10:14:19.516 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : elfutils-libelf-devel-0.186-1.el8.x86_64 91/134 2023-03-27T10:14:19.516 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libuuid-devel-2.32.1-35.el8.x86_64 92/134 2023-03-27T10:14:19.517 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : zlib-devel-1.2.11-18.el8_5.x86_64 93/134 2023-03-27T10:14:19.517 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libcom_err-devel-1.45.6-4.el8.x86_64 94/134 2023-03-27T10:14:19.517 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 95/134 2023-03-27T10:14:19.517 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 96/134 2023-03-27T10:14:19.517 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : NetworkManager-initscripts-updown-1:1.36.0-9.el8 97/134 2023-03-27T10:14:19.517 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd 98/134 2023-03-27T10:14:19.517 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 99/134 2023-03-27T10:14:19.518 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 100/134 2023-03-27T10:14:19.518 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 101/134 2023-03-27T10:14:19.518 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 102/134 2023-03-27T10:14:19.518 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 103/134 2023-03-27T10:14:19.518 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : bison-3.0.4-10.el8.x86_64 104/134 2023-03-27T10:14:19.518 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 105/134 2023-03-27T10:14:19.518 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 106/134 2023-03-27T10:14:19.518 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 107/134 2023-03-27T10:14:19.519 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : automake-1.16.1-7.el8.noarch 108/134 2023-03-27T10:14:19.519 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 109/134 2023-03-27T10:14:19.519 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 110/134 2023-03-27T10:14:19.519 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : flex-2.6.1-9.el8.x86_64 111/134 2023-03-27T10:14:19.519 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 112/134 2023-03-27T10:14:19.519 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 113/134 2023-03-27T10:14:19.519 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 114/134 2023-03-27T10:14:19.519 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 115/134 2023-03-27T10:14:19.520 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : uuid-1.6.2-43.el8.x86_64 116/134 2023-03-27T10:14:19.520 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 117/134 2023-03-27T10:14:19.520 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 118/134 2023-03-27T10:14:19.520 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 119/134 2023-03-27T10:14:19.520 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libtool-2.4.6-25.el8.x86_64 120/134 2023-03-27T10:14:19.520 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 121/134 2023-03-27T10:14:19.520 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libpq-13.5-1.el8.x86_64 122/134 2023-03-27T10:14:19.521 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 123/134 2023-03-27T10:14:19.521 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : jq-1.6-3.el8.x86_64 124/134 2023-03-27T10:14:19.521 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 125/134 2023-03-27T10:14:19.521 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : postgresql-server-10.21-2.module+el8.6.0+15342+5 126/134 2023-03-27T10:14:19.521 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : postgresql-contrib-10.21-2.module+el8.6.0+15342+ 127/134 2023-03-27T10:14:19.521 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : postgresql-10.21-2.module+el8.6.0+15342+53518fac 128/134 2023-03-27T10:14:19.521 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librados2-2:18.0.0-2533.gb70fdf97.el8.x86_64 129/134 2023-03-27T10:14:19.522 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 130/134 2023-03-27T10:14:19.522 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librbd1-2:18.0.0-2533.gb70fdf97.el8.x86_64 131/134 2023-03-27T10:14:19.522 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-03-27T10:14:19.522 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : iproute-5.15.0-4.el8_6.1.x86_64 133/134 2023-03-27T10:14:19.594 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-03-27T10:14:19.611 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-03-27T10:14:19.668 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libtool-2.4.6-25.el8.x86_64 116/134 2023-03-27T10:14:19.850 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : cephfs-top-2:18.0.0-2533.gb70fdf97.el8.noarch 124/134 2023-03-27T10:14:19.860 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: libtool-2.4.6-25.el8.x86_64 116/134 2023-03-27T10:14:20.102 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : rbd-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 125/134 2023-03-27T10:14:20.429 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : rbd-nbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 126/134 2023-03-27T10:14:20.768 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : postgresql-contrib-10.21-2.module+el8.6.0+15342+ 117/134 2023-03-27T10:14:20.790 INFO:teuthology.orchestra.run.smithi161.stdout: Upgrading : iproute-5.15.0-4.el8_6.1.x86_64 127/134 2023-03-27T10:14:20.985 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : flex-2.6.1-9.el8.x86_64 128/134 2023-03-27T10:14:21.072 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: flex-2.6.1-9.el8.x86_64 128/134 2023-03-27T10:14:21.132 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : openssl-devel-1:1.1.1k-7.el8_6.x86_64 118/134 2023-03-27T10:14:21.302 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : bison-3.0.4-10.el8.x86_64 129/134 2023-03-27T10:14:21.315 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: bison-3.0.4-10.el8.x86_64 129/134 2023-03-27T10:14:21.346 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 119/134 2023-03-27T10:14:21.519 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : NetworkManager-initscripts-updown-1:1.36.0-9.el8 130/134 2023-03-27T10:14:21.527 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: NetworkManager-initscripts-updown-1:1.36.0-9.el8 130/134 2023-03-27T10:14:21.580 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : xfsprogs-devel-5.0.0-10.el8.x86_64 120/134 2023-03-27T10:14:21.663 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : libaio-devel-0.3.112-1.el8.x86_64 131/134 2023-03-27T10:14:21.663 INFO:teuthology.orchestra.run.smithi161.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-03-27T10:14:21.780 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libcephfs-devel-2:18.0.0-2533.gb70fdf97.el8.x86_ 121/134 2023-03-27T10:14:21.889 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-03-27T10:14:21.889 INFO:teuthology.orchestra.run.smithi161.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 133/134 2023-03-27T10:14:21.936 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : elfutils-libelf-devel-0.186-1.el8.x86_64 122/134 2023-03-27T10:14:21.965 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : iproute-5.15.0-4.el8.x86_64 134/134 2023-03-27T10:14:21.966 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:52 ago on Mon 27 Mar 2023 10:12:29 AM UTC. 2023-03-27T10:14:22.141 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 133/134 2023-03-27T10:14:22.376 INFO:teuthology.orchestra.run.smithi161.stdout: Cleanup : iproute-5.15.0-4.el8.x86_64 134/134 2023-03-27T10:14:22.662 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: iproute-5.15.0-4.el8.x86_64 134/134 2023-03-27T10:14:22.662 INFO:teuthology.orchestra.run.smithi161.stdout:[/usr/lib/tmpfiles.d/journal-nocow.conf:26] Failed to resolve specifier: uninitialized /etc detected, skipping 2023-03-27T10:14:22.662 INFO:teuthology.orchestra.run.smithi161.stdout:All rules containing unresolvable specifiers will be skipped. 2023-03-27T10:14:22.663 INFO:teuthology.orchestra.run.smithi161.stdout: 2023-03-27T10:14:22.882 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-2:18.0.0-2533.gb70fdf97.el8.x86_64 1/134 2023-03-27T10:14:22.882 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-base-2:18.0.0-2533.gb70fdf97.el8.x86_64 2/134 2023-03-27T10:14:22.882 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 3/134 2023-03-27T10:14:22.882 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 4/134 2023-03-27T10:14:22.882 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-immutable-object-cache-2:18.0.0-2533.gb70fd 5/134 2023-03-27T10:14:22.883 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-mds-2:18.0.0-2533.gb70fdf97.el8.x86_64 6/134 2023-03-27T10:14:22.883 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-mgr-2:18.0.0-2533.gb70fdf97.el8.x86_64 7/134 2023-03-27T10:14:22.883 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-mon-2:18.0.0-2533.gb70fdf97.el8.x86_64 8/134 2023-03-27T10:14:22.883 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-osd-2:18.0.0-2533.gb70fdf97.el8.x86_64 9/134 2023-03-27T10:14:22.883 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-radosgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 10/134 2023-03-27T10:14:22.883 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-selinux-2:18.0.0-2533.gb70fdf97.el8.x86_64 11/134 2023-03-27T10:14:22.883 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-test-2:18.0.0-2533.gb70fdf97.el8.x86_64 12/134 2023-03-27T10:14:22.883 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : cephfs-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 13/134 2023-03-27T10:14:22.884 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : libcephfs-devel-2:18.0.0-2533.gb70fdf97.el8.x86_ 14/134 2023-03-27T10:14:22.884 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : libcephfs2-2:18.0.0-2533.gb70fdf97.el8.x86_64 15/134 2023-03-27T10:14:22.884 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : libcephsqlite-2:18.0.0-2533.gb70fdf97.el8.x86_64 16/134 2023-03-27T10:14:22.884 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : librados-devel-2:18.0.0-2533.gb70fdf97.el8.x86_6 17/134 2023-03-27T10:14:22.884 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : libradosstriper1-2:18.0.0-2533.gb70fdf97.el8.x86 18/134 2023-03-27T10:14:22.884 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : librgw2-2:18.0.0-2533.gb70fdf97.el8.x86_64 19/134 2023-03-27T10:14:22.884 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-ceph-argparse-2:18.0.0-2533.gb70fdf97.el 20/134 2023-03-27T10:14:22.884 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-ceph-common-2:18.0.0-2533.gb70fdf97.el8. 21/134 2023-03-27T10:14:22.885 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-cephfs-2:18.0.0-2533.gb70fdf97.el8.x86_6 22/134 2023-03-27T10:14:22.885 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-rados-2:18.0.0-2533.gb70fdf97.el8.x86_64 23/134 2023-03-27T10:14:22.885 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-rbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 24/134 2023-03-27T10:14:22.892 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-rgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 25/134 2023-03-27T10:14:22.892 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : rbd-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 26/134 2023-03-27T10:14:22.893 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : rbd-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 27/134 2023-03-27T10:14:22.893 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : rbd-nbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 28/134 2023-03-27T10:14:22.893 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-grafana-dashboards-2:18.0.0-2533.gb70fdf97. 29/134 2023-03-27T10:14:22.893 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-mgr-cephadm-2:18.0.0-2533.gb70fdf97.el8.noa 30/134 2023-03-27T10:14:22.893 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-mgr-dashboard-2:18.0.0-2533.gb70fdf97.el8.n 31/134 2023-03-27T10:14:22.893 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.0.0-2533.gb70 32/134 2023-03-27T10:14:22.893 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-mgr-modules-core-2:18.0.0-2533.gb70fdf97.el 33/134 2023-03-27T10:14:22.894 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-mgr-rook-2:18.0.0-2533.gb70fdf97.el8.noarch 34/134 2023-03-27T10:14:22.894 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-prometheus-alerts-2:18.0.0-2533.gb70fdf97.e 35/134 2023-03-27T10:14:22.894 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : ceph-volume-2:18.0.0-2533.gb70fdf97.el8.noarch 36/134 2023-03-27T10:14:22.894 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : cephadm-2:18.0.0-2533.gb70fdf97.el8.noarch 37/134 2023-03-27T10:14:22.894 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : cephfs-top-2:18.0.0-2533.gb70fdf97.el8.noarch 38/134 2023-03-27T10:14:22.894 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 39/134 2023-03-27T10:14:22.894 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 40/134 2023-03-27T10:14:22.895 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 41/134 2023-03-27T10:14:22.895 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 42/134 2023-03-27T10:14:22.895 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 43/134 2023-03-27T10:14:22.895 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 44/134 2023-03-27T10:14:22.895 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 45/134 2023-03-27T10:14:22.895 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 46/134 2023-03-27T10:14:22.895 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 47/134 2023-03-27T10:14:22.895 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 48/134 2023-03-27T10:14:22.896 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 49/134 2023-03-27T10:14:22.896 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 50/134 2023-03-27T10:14:22.896 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 51/134 2023-03-27T10:14:22.896 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 52/134 2023-03-27T10:14:22.896 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 53/134 2023-03-27T10:14:22.896 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 54/134 2023-03-27T10:14:22.896 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 55/134 2023-03-27T10:14:22.896 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 56/134 2023-03-27T10:14:22.897 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-portend-2.6-1.el8.noarch 57/134 2023-03-27T10:14:22.897 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 58/134 2023-03-27T10:14:22.897 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 59/134 2023-03-27T10:14:22.897 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-rsa-4.8-1.el8.noarch 60/134 2023-03-27T10:14:22.897 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 61/134 2023-03-27T10:14:22.897 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 62/134 2023-03-27T10:14:22.897 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 63/134 2023-03-27T10:14:22.898 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 64/134 2023-03-27T10:14:22.898 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-waitress-1.2.1-2.el8.1.noarch 65/134 2023-03-27T10:14:22.898 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 66/134 2023-03-27T10:14:22.898 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 67/134 2023-03-27T10:14:22.898 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 68/134 2023-03-27T10:14:22.898 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 69/134 2023-03-27T10:14:22.898 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 70/134 2023-03-27T10:14:22.898 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 71/134 2023-03-27T10:14:22.899 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 72/134 2023-03-27T10:14:22.899 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : libselinux-devel-2.9-5.el8.x86_64 73/134 2023-03-27T10:14:22.899 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 74/134 2023-03-27T10:14:22.899 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : pcre2-utf32-10.32-2.el8.x86_64 75/134 2023-03-27T10:14:22.899 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : keyutils-libs-devel-1.5.10-9.el8.x86_64 76/134 2023-03-27T10:14:22.899 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : libsepol-devel-2.9-3.el8.x86_64 77/134 2023-03-27T10:14:22.900 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 78/134 2023-03-27T10:14:22.900 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 79/134 2023-03-27T10:14:22.900 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : pcre2-utf16-10.32-2.el8.x86_64 80/134 2023-03-27T10:14:22.900 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : krb5-devel-1.18.2-14.el8.x86_64 81/134 2023-03-27T10:14:22.900 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : libaio-devel-0.3.112-1.el8.x86_64 82/134 2023-03-27T10:14:22.900 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 83/134 2023-03-27T10:14:22.900 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 84/134 2023-03-27T10:14:22.900 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : libverto-devel-0.3.0-5.el8.x86_64 85/134 2023-03-27T10:14:22.901 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : pcre2-devel-10.32-2.el8.x86_64 86/134 2023-03-27T10:14:22.901 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 87/134 2023-03-27T10:14:22.901 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : openssl-devel-1:1.1.1k-7.el8_6.x86_64 88/134 2023-03-27T10:14:22.901 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : xfsprogs-devel-5.0.0-10.el8.x86_64 89/134 2023-03-27T10:14:22.901 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 90/134 2023-03-27T10:14:22.901 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : elfutils-libelf-devel-0.186-1.el8.x86_64 91/134 2023-03-27T10:14:22.901 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : libuuid-devel-2.32.1-35.el8.x86_64 92/134 2023-03-27T10:14:22.902 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : zlib-devel-1.2.11-18.el8_5.x86_64 93/134 2023-03-27T10:14:22.902 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : libcom_err-devel-1.45.6-4.el8.x86_64 94/134 2023-03-27T10:14:22.902 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 95/134 2023-03-27T10:14:22.902 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 96/134 2023-03-27T10:14:22.902 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : NetworkManager-initscripts-updown-1:1.36.0-9.el8 97/134 2023-03-27T10:14:22.902 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd 98/134 2023-03-27T10:14:22.902 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 99/134 2023-03-27T10:14:22.902 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 100/134 2023-03-27T10:14:22.903 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 101/134 2023-03-27T10:14:22.903 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 102/134 2023-03-27T10:14:22.903 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 103/134 2023-03-27T10:14:22.903 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : bison-3.0.4-10.el8.x86_64 104/134 2023-03-27T10:14:22.903 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 105/134 2023-03-27T10:14:22.903 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 106/134 2023-03-27T10:14:22.904 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 107/134 2023-03-27T10:14:22.904 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : automake-1.16.1-7.el8.noarch 108/134 2023-03-27T10:14:22.904 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 109/134 2023-03-27T10:14:22.904 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 110/134 2023-03-27T10:14:22.904 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : flex-2.6.1-9.el8.x86_64 111/134 2023-03-27T10:14:22.904 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 112/134 2023-03-27T10:14:22.904 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 113/134 2023-03-27T10:14:22.904 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 114/134 2023-03-27T10:14:22.905 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 115/134 2023-03-27T10:14:22.905 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : uuid-1.6.2-43.el8.x86_64 116/134 2023-03-27T10:14:22.905 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 117/134 2023-03-27T10:14:22.905 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 118/134 2023-03-27T10:14:22.905 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 119/134 2023-03-27T10:14:22.905 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : libtool-2.4.6-25.el8.x86_64 120/134 2023-03-27T10:14:22.905 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 121/134 2023-03-27T10:14:22.905 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : libpq-13.5-1.el8.x86_64 122/134 2023-03-27T10:14:22.906 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 123/134 2023-03-27T10:14:22.906 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : jq-1.6-3.el8.x86_64 124/134 2023-03-27T10:14:22.906 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 125/134 2023-03-27T10:14:22.906 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : postgresql-server-10.21-2.module+el8.6.0+15342+5 126/134 2023-03-27T10:14:22.906 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : postgresql-contrib-10.21-2.module+el8.6.0+15342+ 127/134 2023-03-27T10:14:22.906 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : postgresql-10.21-2.module+el8.6.0+15342+53518fac 128/134 2023-03-27T10:14:22.906 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : librados2-2:18.0.0-2533.gb70fdf97.el8.x86_64 129/134 2023-03-27T10:14:22.907 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 130/134 2023-03-27T10:14:22.907 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : librbd1-2:18.0.0-2533.gb70fdf97.el8.x86_64 131/134 2023-03-27T10:14:22.907 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-03-27T10:14:22.907 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : iproute-5.15.0-4.el8_6.1.x86_64 133/134 2023-03-27T10:14:24.165 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-03-27T10:14:24.484 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-03-27T10:14:24.487 INFO:teuthology.orchestra.run.smithi171.stdout:Installed products updated. 2023-03-27T10:14:24.503 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-03-27T10:14:24.696 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:14:24.696 INFO:teuthology.orchestra.run.smithi171.stdout:Upgraded: 2023-03-27T10:14:24.696 INFO:teuthology.orchestra.run.smithi171.stdout: iproute-5.15.0-4.el8_6.1.x86_64 2023-03-27T10:14:24.696 INFO:teuthology.orchestra.run.smithi171.stdout: librados2-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.696 INFO:teuthology.orchestra.run.smithi171.stdout: librbd1-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.697 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2023-03-27T10:14:24.697 INFO:teuthology.orchestra.run.smithi171.stdout: NetworkManager-initscripts-updown-1:1.36.0-9.el8_6.noarch 2023-03-27T10:14:24.697 INFO:teuthology.orchestra.run.smithi171.stdout: automake-1.16.1-7.el8.noarch 2023-03-27T10:14:24.697 INFO:teuthology.orchestra.run.smithi171.stdout: bison-3.0.4-10.el8.x86_64 2023-03-27T10:14:24.697 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.697 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-base-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.697 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.698 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.698 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-grafana-dashboards-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:24.698 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-immutable-object-cache-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.698 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mds-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.698 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.698 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-cephadm-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:24.698 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-dashboard-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:24.699 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-diskprediction-local-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:24.699 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-modules-core-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:24.699 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-rook-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:24.699 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mon-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.699 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-osd-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.699 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-prometheus-alerts-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:24.699 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-radosgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.699 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-selinux-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.700 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-test-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.700 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-volume-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:24.700 INFO:teuthology.orchestra.run.smithi171.stdout: cephadm-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:24.700 INFO:teuthology.orchestra.run.smithi171.stdout: cephfs-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.700 INFO:teuthology.orchestra.run.smithi171.stdout: cephfs-top-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:24.700 INFO:teuthology.orchestra.run.smithi171.stdout: elfutils-libelf-devel-0.186-1.el8.x86_64 2023-03-27T10:14:24.700 INFO:teuthology.orchestra.run.smithi171.stdout: flex-2.6.1-9.el8.x86_64 2023-03-27T10:14:24.700 INFO:teuthology.orchestra.run.smithi171.stdout: fuse-2.9.7-15.el8.x86_64 2023-03-27T10:14:24.701 INFO:teuthology.orchestra.run.smithi171.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-03-27T10:14:24.701 INFO:teuthology.orchestra.run.smithi171.stdout: jq-1.6-3.el8.x86_64 2023-03-27T10:14:24.701 INFO:teuthology.orchestra.run.smithi171.stdout: keyutils-libs-devel-1.5.10-9.el8.x86_64 2023-03-27T10:14:24.701 INFO:teuthology.orchestra.run.smithi171.stdout: krb5-devel-1.18.2-14.el8.x86_64 2023-03-27T10:14:24.701 INFO:teuthology.orchestra.run.smithi171.stdout: libaio-devel-0.3.112-1.el8.x86_64 2023-03-27T10:14:24.701 INFO:teuthology.orchestra.run.smithi171.stdout: libcephfs-devel-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.701 INFO:teuthology.orchestra.run.smithi171.stdout: libcephfs2-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.702 INFO:teuthology.orchestra.run.smithi171.stdout: libcephsqlite-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.702 INFO:teuthology.orchestra.run.smithi171.stdout: libcom_err-devel-1.45.6-4.el8.x86_64 2023-03-27T10:14:24.702 INFO:teuthology.orchestra.run.smithi171.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-03-27T10:14:24.702 INFO:teuthology.orchestra.run.smithi171.stdout: liboath-2.6.2-3.el8.x86_64 2023-03-27T10:14:24.702 INFO:teuthology.orchestra.run.smithi171.stdout: libpq-13.5-1.el8.x86_64 2023-03-27T10:14:24.702 INFO:teuthology.orchestra.run.smithi171.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-03-27T10:14:24.702 INFO:teuthology.orchestra.run.smithi171.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-03-27T10:14:24.702 INFO:teuthology.orchestra.run.smithi171.stdout: librados-devel-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.703 INFO:teuthology.orchestra.run.smithi171.stdout: libradosstriper1-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.703 INFO:teuthology.orchestra.run.smithi171.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-03-27T10:14:24.703 INFO:teuthology.orchestra.run.smithi171.stdout: librgw2-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.703 INFO:teuthology.orchestra.run.smithi171.stdout: libselinux-devel-2.9-5.el8.x86_64 2023-03-27T10:14:24.703 INFO:teuthology.orchestra.run.smithi171.stdout: libsepol-devel-2.9-3.el8.x86_64 2023-03-27T10:14:24.703 INFO:teuthology.orchestra.run.smithi171.stdout: libtool-2.4.6-25.el8.x86_64 2023-03-27T10:14:24.703 INFO:teuthology.orchestra.run.smithi171.stdout: libunwind-1.3.1-3.el8.x86_64 2023-03-27T10:14:24.703 INFO:teuthology.orchestra.run.smithi171.stdout: libuuid-devel-2.32.1-35.el8.x86_64 2023-03-27T10:14:24.704 INFO:teuthology.orchestra.run.smithi171.stdout: libverto-devel-0.3.0-5.el8.x86_64 2023-03-27T10:14:24.704 INFO:teuthology.orchestra.run.smithi171.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-03-27T10:14:24.704 INFO:teuthology.orchestra.run.smithi171.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-03-27T10:14:24.704 INFO:teuthology.orchestra.run.smithi171.stdout: openblas-0.3.15-3.el8.x86_64 2023-03-27T10:14:24.704 INFO:teuthology.orchestra.run.smithi171.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-03-27T10:14:24.704 INFO:teuthology.orchestra.run.smithi171.stdout: openssl-devel-1:1.1.1k-7.el8_6.x86_64 2023-03-27T10:14:24.704 INFO:teuthology.orchestra.run.smithi171.stdout: pcre2-devel-10.32-2.el8.x86_64 2023-03-27T10:14:24.704 INFO:teuthology.orchestra.run.smithi171.stdout: pcre2-utf16-10.32-2.el8.x86_64 2023-03-27T10:14:24.705 INFO:teuthology.orchestra.run.smithi171.stdout: pcre2-utf32-10.32-2.el8.x86_64 2023-03-27T10:14:24.705 INFO:teuthology.orchestra.run.smithi171.stdout: postgresql-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-03-27T10:14:24.705 INFO:teuthology.orchestra.run.smithi171.stdout: postgresql-contrib-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-03-27T10:14:24.705 INFO:teuthology.orchestra.run.smithi171.stdout: postgresql-server-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-03-27T10:14:24.705 INFO:teuthology.orchestra.run.smithi171.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-03-27T10:14:24.705 INFO:teuthology.orchestra.run.smithi171.stdout: python3-babel-2.5.1-7.el8.noarch 2023-03-27T10:14:24.705 INFO:teuthology.orchestra.run.smithi171.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-03-27T10:14:24.705 INFO:teuthology.orchestra.run.smithi171.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-03-27T10:14:24.706 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-03-27T10:14:24.706 INFO:teuthology.orchestra.run.smithi171.stdout: python3-ceph-argparse-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.706 INFO:teuthology.orchestra.run.smithi171.stdout: python3-ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.706 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cephfs-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.706 INFO:teuthology.orchestra.run.smithi171.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-03-27T10:14:24.706 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-03-27T10:14:24.706 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-03-27T10:14:24.706 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-03-27T10:14:24.707 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-03-27T10:14:24.707 INFO:teuthology.orchestra.run.smithi171.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-03-27T10:14:24.707 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-6.2-6.el8.noarch 2023-03-27T10:14:24.707 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-03-27T10:14:24.707 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-03-27T10:14:24.707 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-03-27T10:14:24.707 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-03-27T10:14:24.708 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-03-27T10:14:24.708 INFO:teuthology.orchestra.run.smithi171.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-03-27T10:14:24.708 INFO:teuthology.orchestra.run.smithi171.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-03-27T10:14:24.708 INFO:teuthology.orchestra.run.smithi171.stdout: python3-mako-1.0.6-13.el8.noarch 2023-03-27T10:14:24.708 INFO:teuthology.orchestra.run.smithi171.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-03-27T10:14:24.708 INFO:teuthology.orchestra.run.smithi171.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-03-27T10:14:24.708 INFO:teuthology.orchestra.run.smithi171.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-03-27T10:14:24.708 INFO:teuthology.orchestra.run.smithi171.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-03-27T10:14:24.709 INFO:teuthology.orchestra.run.smithi171.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-03-27T10:14:24.709 INFO:teuthology.orchestra.run.smithi171.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-03-27T10:14:24.709 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-03-27T10:14:24.709 INFO:teuthology.orchestra.run.smithi171.stdout: python3-portend-2.6-1.el8.noarch 2023-03-27T10:14:24.709 INFO:teuthology.orchestra.run.smithi171.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-03-27T10:14:24.709 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-03-27T10:14:24.709 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-03-27T10:14:24.709 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-03-27T10:14:24.710 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pycparser-2.14-14.el8.noarch 2023-03-27T10:14:24.710 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pytz-2017.2-9.el8.noarch 2023-03-27T10:14:24.710 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rados-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.710 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.710 INFO:teuthology.orchestra.run.smithi171.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-03-27T10:14:24.710 INFO:teuthology.orchestra.run.smithi171.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-03-27T10:14:24.710 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.711 INFO:teuthology.orchestra.run.smithi171.stdout: python3-routes-2.4.1-12.el8.noarch 2023-03-27T10:14:24.711 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rsa-4.8-1.el8.noarch 2023-03-27T10:14:24.711 INFO:teuthology.orchestra.run.smithi171.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-03-27T10:14:24.711 INFO:teuthology.orchestra.run.smithi171.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-03-27T10:14:24.711 INFO:teuthology.orchestra.run.smithi171.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-03-27T10:14:24.711 INFO:teuthology.orchestra.run.smithi171.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-03-27T10:14:24.711 INFO:teuthology.orchestra.run.smithi171.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-03-27T10:14:24.711 INFO:teuthology.orchestra.run.smithi171.stdout: python3-waitress-1.2.1-2.el8.1.noarch 2023-03-27T10:14:24.712 INFO:teuthology.orchestra.run.smithi171.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-03-27T10:14:24.712 INFO:teuthology.orchestra.run.smithi171.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-03-27T10:14:24.712 INFO:teuthology.orchestra.run.smithi171.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-03-27T10:14:24.712 INFO:teuthology.orchestra.run.smithi171.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-03-27T10:14:24.712 INFO:teuthology.orchestra.run.smithi171.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-03-27T10:14:24.712 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.712 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.712 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-nbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:24.713 INFO:teuthology.orchestra.run.smithi171.stdout: socat-1.7.4.1-1.el8.x86_64 2023-03-27T10:14:24.713 INFO:teuthology.orchestra.run.smithi171.stdout: thrift-0.13.0-2.el8.x86_64 2023-03-27T10:14:24.713 INFO:teuthology.orchestra.run.smithi171.stdout: uuid-1.6.2-43.el8.x86_64 2023-03-27T10:14:24.713 INFO:teuthology.orchestra.run.smithi171.stdout: xfsprogs-devel-5.0.0-10.el8.x86_64 2023-03-27T10:14:24.713 INFO:teuthology.orchestra.run.smithi171.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-03-27T10:14:24.714 INFO:teuthology.orchestra.run.smithi171.stdout: zlib-devel-1.2.11-18.el8_5.x86_64 2023-03-27T10:14:24.714 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:14:24.714 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2023-03-27T10:14:24.793 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : cephfs-top-2:18.0.0-2533.gb70fdf97.el8.noarch 124/134 2023-03-27T10:14:24.910 DEBUG:teuthology.parallel:result is None 2023-03-27T10:14:25.042 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : rbd-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 125/134 2023-03-27T10:14:25.333 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : rbd-nbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 126/134 2023-03-27T10:14:25.462 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : iproute-5.15.0-4.el8.x86_64 134/134 2023-03-27T10:14:25.463 INFO:teuthology.orchestra.run.smithi161.stdout:Last metadata expiration check: 0:01:56 ago on Mon 27 Mar 2023 10:12:29 AM UTC. 2023-03-27T10:14:25.652 INFO:teuthology.orchestra.run.smithi159.stdout: Upgrading : iproute-5.15.0-4.el8_6.1.x86_64 127/134 2023-03-27T10:14:25.817 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : NetworkManager-initscripts-updown-1:1.36.0-9.el8 128/134 2023-03-27T10:14:25.825 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: NetworkManager-initscripts-updown-1:1.36.0-9.el8 128/134 2023-03-27T10:14:26.060 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libaio-devel-0.3.112-1.el8.x86_64 129/134 2023-03-27T10:14:26.284 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : flex-2.6.1-9.el8.x86_64 130/134 2023-03-27T10:14:26.364 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: flex-2.6.1-9.el8.x86_64 130/134 2023-03-27T10:14:26.592 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : bison-3.0.4-10.el8.x86_64 131/134 2023-03-27T10:14:26.604 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: bison-3.0.4-10.el8.x86_64 131/134 2023-03-27T10:14:26.605 INFO:teuthology.orchestra.run.smithi159.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-03-27T10:14:26.837 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-03-27T10:14:26.837 INFO:teuthology.orchestra.run.smithi159.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 133/134 2023-03-27T10:14:27.122 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 133/134 2023-03-27T10:14:27.365 INFO:teuthology.orchestra.run.smithi159.stdout: Cleanup : iproute-5.15.0-4.el8.x86_64 134/134 2023-03-27T10:14:27.641 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: iproute-5.15.0-4.el8.x86_64 134/134 2023-03-27T10:14:27.641 INFO:teuthology.orchestra.run.smithi159.stdout:[/usr/lib/tmpfiles.d/journal-nocow.conf:26] Failed to resolve specifier: uninitialized /etc detected, skipping 2023-03-27T10:14:27.642 INFO:teuthology.orchestra.run.smithi159.stdout:All rules containing unresolvable specifiers will be skipped. 2023-03-27T10:14:27.642 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:14:27.880 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-2:18.0.0-2533.gb70fdf97.el8.x86_64 1/134 2023-03-27T10:14:27.880 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-base-2:18.0.0-2533.gb70fdf97.el8.x86_64 2/134 2023-03-27T10:14:27.880 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 3/134 2023-03-27T10:14:27.880 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 4/134 2023-03-27T10:14:27.880 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-immutable-object-cache-2:18.0.0-2533.gb70fd 5/134 2023-03-27T10:14:27.880 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-mds-2:18.0.0-2533.gb70fdf97.el8.x86_64 6/134 2023-03-27T10:14:27.881 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-mgr-2:18.0.0-2533.gb70fdf97.el8.x86_64 7/134 2023-03-27T10:14:27.881 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-mon-2:18.0.0-2533.gb70fdf97.el8.x86_64 8/134 2023-03-27T10:14:27.881 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-osd-2:18.0.0-2533.gb70fdf97.el8.x86_64 9/134 2023-03-27T10:14:27.881 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-radosgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 10/134 2023-03-27T10:14:27.881 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-selinux-2:18.0.0-2533.gb70fdf97.el8.x86_64 11/134 2023-03-27T10:14:27.881 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-test-2:18.0.0-2533.gb70fdf97.el8.x86_64 12/134 2023-03-27T10:14:27.882 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : cephfs-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 13/134 2023-03-27T10:14:27.882 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libcephfs-devel-2:18.0.0-2533.gb70fdf97.el8.x86_ 14/134 2023-03-27T10:14:27.882 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libcephfs2-2:18.0.0-2533.gb70fdf97.el8.x86_64 15/134 2023-03-27T10:14:27.882 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libcephsqlite-2:18.0.0-2533.gb70fdf97.el8.x86_64 16/134 2023-03-27T10:14:27.882 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : librados-devel-2:18.0.0-2533.gb70fdf97.el8.x86_6 17/134 2023-03-27T10:14:27.882 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libradosstriper1-2:18.0.0-2533.gb70fdf97.el8.x86 18/134 2023-03-27T10:14:27.882 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : librgw2-2:18.0.0-2533.gb70fdf97.el8.x86_64 19/134 2023-03-27T10:14:27.882 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-ceph-argparse-2:18.0.0-2533.gb70fdf97.el 20/134 2023-03-27T10:14:27.883 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-ceph-common-2:18.0.0-2533.gb70fdf97.el8. 21/134 2023-03-27T10:14:27.883 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-cephfs-2:18.0.0-2533.gb70fdf97.el8.x86_6 22/134 2023-03-27T10:14:27.883 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-rados-2:18.0.0-2533.gb70fdf97.el8.x86_64 23/134 2023-03-27T10:14:27.883 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-rbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 24/134 2023-03-27T10:14:27.883 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-rgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 25/134 2023-03-27T10:14:27.890 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : rbd-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 26/134 2023-03-27T10:14:27.890 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : rbd-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 27/134 2023-03-27T10:14:27.890 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : rbd-nbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 28/134 2023-03-27T10:14:27.890 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-grafana-dashboards-2:18.0.0-2533.gb70fdf97. 29/134 2023-03-27T10:14:27.891 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-mgr-cephadm-2:18.0.0-2533.gb70fdf97.el8.noa 30/134 2023-03-27T10:14:27.891 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-mgr-dashboard-2:18.0.0-2533.gb70fdf97.el8.n 31/134 2023-03-27T10:14:27.891 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.0.0-2533.gb70 32/134 2023-03-27T10:14:27.891 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-mgr-modules-core-2:18.0.0-2533.gb70fdf97.el 33/134 2023-03-27T10:14:27.891 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-mgr-rook-2:18.0.0-2533.gb70fdf97.el8.noarch 34/134 2023-03-27T10:14:27.891 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-prometheus-alerts-2:18.0.0-2533.gb70fdf97.e 35/134 2023-03-27T10:14:27.891 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-volume-2:18.0.0-2533.gb70fdf97.el8.noarch 36/134 2023-03-27T10:14:27.891 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : cephadm-2:18.0.0-2533.gb70fdf97.el8.noarch 37/134 2023-03-27T10:14:27.892 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : cephfs-top-2:18.0.0-2533.gb70fdf97.el8.noarch 38/134 2023-03-27T10:14:27.892 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 39/134 2023-03-27T10:14:27.892 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 40/134 2023-03-27T10:14:27.892 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 41/134 2023-03-27T10:14:27.892 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 42/134 2023-03-27T10:14:27.892 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 43/134 2023-03-27T10:14:27.892 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 44/134 2023-03-27T10:14:27.892 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 45/134 2023-03-27T10:14:27.893 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 46/134 2023-03-27T10:14:27.893 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 47/134 2023-03-27T10:14:27.893 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 48/134 2023-03-27T10:14:27.893 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 49/134 2023-03-27T10:14:27.893 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 50/134 2023-03-27T10:14:27.893 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 51/134 2023-03-27T10:14:27.893 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 52/134 2023-03-27T10:14:27.894 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 53/134 2023-03-27T10:14:27.894 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 54/134 2023-03-27T10:14:27.894 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 55/134 2023-03-27T10:14:27.894 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 56/134 2023-03-27T10:14:27.894 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-portend-2.6-1.el8.noarch 57/134 2023-03-27T10:14:27.894 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 58/134 2023-03-27T10:14:27.894 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 59/134 2023-03-27T10:14:27.894 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-rsa-4.8-1.el8.noarch 60/134 2023-03-27T10:14:27.895 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 61/134 2023-03-27T10:14:27.895 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 62/134 2023-03-27T10:14:27.895 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 63/134 2023-03-27T10:14:27.895 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 64/134 2023-03-27T10:14:27.895 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-waitress-1.2.1-2.el8.1.noarch 65/134 2023-03-27T10:14:27.895 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 66/134 2023-03-27T10:14:27.895 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 67/134 2023-03-27T10:14:27.895 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 68/134 2023-03-27T10:14:27.896 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 69/134 2023-03-27T10:14:27.896 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 70/134 2023-03-27T10:14:27.896 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 71/134 2023-03-27T10:14:27.896 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd 72/134 2023-03-27T10:14:27.896 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 73/134 2023-03-27T10:14:27.896 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 74/134 2023-03-27T10:14:27.896 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 75/134 2023-03-27T10:14:27.896 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 76/134 2023-03-27T10:14:27.897 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 77/134 2023-03-27T10:14:27.897 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : bison-3.0.4-10.el8.x86_64 78/134 2023-03-27T10:14:27.897 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 79/134 2023-03-27T10:14:27.897 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 80/134 2023-03-27T10:14:27.897 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 81/134 2023-03-27T10:14:27.897 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : automake-1.16.1-7.el8.noarch 82/134 2023-03-27T10:14:27.897 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 83/134 2023-03-27T10:14:27.898 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 84/134 2023-03-27T10:14:27.898 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : flex-2.6.1-9.el8.x86_64 85/134 2023-03-27T10:14:27.898 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 86/134 2023-03-27T10:14:27.898 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 87/134 2023-03-27T10:14:27.898 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 88/134 2023-03-27T10:14:27.898 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 89/134 2023-03-27T10:14:27.898 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : uuid-1.6.2-43.el8.x86_64 90/134 2023-03-27T10:14:27.898 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 91/134 2023-03-27T10:14:27.899 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 92/134 2023-03-27T10:14:27.899 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 93/134 2023-03-27T10:14:27.899 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libtool-2.4.6-25.el8.x86_64 94/134 2023-03-27T10:14:27.899 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 95/134 2023-03-27T10:14:27.899 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libpq-13.5-1.el8.x86_64 96/134 2023-03-27T10:14:27.899 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 97/134 2023-03-27T10:14:27.899 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : jq-1.6-3.el8.x86_64 98/134 2023-03-27T10:14:27.899 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 99/134 2023-03-27T10:14:27.900 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : postgresql-server-10.21-2.module+el8.6.0+15342+5 100/134 2023-03-27T10:14:27.900 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : postgresql-contrib-10.21-2.module+el8.6.0+15342+ 101/134 2023-03-27T10:14:27.900 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : postgresql-10.21-2.module+el8.6.0+15342+53518fac 102/134 2023-03-27T10:14:27.900 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 103/134 2023-03-27T10:14:27.900 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libselinux-devel-2.9-5.el8.x86_64 104/134 2023-03-27T10:14:27.900 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 105/134 2023-03-27T10:14:27.900 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : pcre2-utf32-10.32-2.el8.x86_64 106/134 2023-03-27T10:14:27.901 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : keyutils-libs-devel-1.5.10-9.el8.x86_64 107/134 2023-03-27T10:14:27.901 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libsepol-devel-2.9-3.el8.x86_64 108/134 2023-03-27T10:14:27.901 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 109/134 2023-03-27T10:14:27.901 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 110/134 2023-03-27T10:14:27.902 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : pcre2-utf16-10.32-2.el8.x86_64 111/134 2023-03-27T10:14:27.902 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : krb5-devel-1.18.2-14.el8.x86_64 112/134 2023-03-27T10:14:27.902 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libaio-devel-0.3.112-1.el8.x86_64 113/134 2023-03-27T10:14:27.902 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 114/134 2023-03-27T10:14:27.903 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 115/134 2023-03-27T10:14:27.903 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libverto-devel-0.3.0-5.el8.x86_64 116/134 2023-03-27T10:14:27.903 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : pcre2-devel-10.32-2.el8.x86_64 117/134 2023-03-27T10:14:27.904 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 118/134 2023-03-27T10:14:27.904 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : openssl-devel-1:1.1.1k-7.el8_6.x86_64 119/134 2023-03-27T10:14:27.904 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : xfsprogs-devel-5.0.0-10.el8.x86_64 120/134 2023-03-27T10:14:27.904 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 121/134 2023-03-27T10:14:27.904 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : elfutils-libelf-devel-0.186-1.el8.x86_64 122/134 2023-03-27T10:14:27.904 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libuuid-devel-2.32.1-35.el8.x86_64 123/134 2023-03-27T10:14:27.905 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : zlib-devel-1.2.11-18.el8_5.x86_64 124/134 2023-03-27T10:14:27.905 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libcom_err-devel-1.45.6-4.el8.x86_64 125/134 2023-03-27T10:14:27.905 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 126/134 2023-03-27T10:14:27.905 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 127/134 2023-03-27T10:14:27.905 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : NetworkManager-initscripts-updown-1:1.36.0-9.el8 128/134 2023-03-27T10:14:27.905 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : librados2-2:18.0.0-2533.gb70fdf97.el8.x86_64 129/134 2023-03-27T10:14:27.905 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 130/134 2023-03-27T10:14:27.905 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : librbd1-2:18.0.0-2533.gb70fdf97.el8.x86_64 131/134 2023-03-27T10:14:27.906 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-03-27T10:14:27.906 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : iproute-5.15.0-4.el8_6.1.x86_64 133/134 2023-03-27T10:14:27.953 INFO:teuthology.orchestra.run.smithi161.stdout:Installed products updated. 2023-03-27T10:14:28.163 INFO:teuthology.orchestra.run.smithi161.stdout: 2023-03-27T10:14:28.163 INFO:teuthology.orchestra.run.smithi161.stdout:Upgraded: 2023-03-27T10:14:28.164 INFO:teuthology.orchestra.run.smithi161.stdout: iproute-5.15.0-4.el8_6.1.x86_64 2023-03-27T10:14:28.164 INFO:teuthology.orchestra.run.smithi161.stdout: librados2-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.164 INFO:teuthology.orchestra.run.smithi161.stdout: librbd1-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.164 INFO:teuthology.orchestra.run.smithi161.stdout:Installed: 2023-03-27T10:14:28.164 INFO:teuthology.orchestra.run.smithi161.stdout: NetworkManager-initscripts-updown-1:1.36.0-9.el8_6.noarch 2023-03-27T10:14:28.164 INFO:teuthology.orchestra.run.smithi161.stdout: automake-1.16.1-7.el8.noarch 2023-03-27T10:14:28.165 INFO:teuthology.orchestra.run.smithi161.stdout: bison-3.0.4-10.el8.x86_64 2023-03-27T10:14:28.165 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.165 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-base-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.165 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.165 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.165 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-grafana-dashboards-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:28.166 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-immutable-object-cache-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.166 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-mds-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.166 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-mgr-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.166 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-mgr-cephadm-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:28.166 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-mgr-dashboard-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:28.166 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-mgr-diskprediction-local-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:28.166 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-mgr-modules-core-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:28.167 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-mgr-rook-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:28.167 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-mon-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.167 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-osd-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.167 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-prometheus-alerts-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:28.167 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-radosgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.167 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-selinux-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.168 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-test-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.168 INFO:teuthology.orchestra.run.smithi161.stdout: ceph-volume-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:28.168 INFO:teuthology.orchestra.run.smithi161.stdout: cephadm-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:28.168 INFO:teuthology.orchestra.run.smithi161.stdout: cephfs-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.168 INFO:teuthology.orchestra.run.smithi161.stdout: cephfs-top-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:28.168 INFO:teuthology.orchestra.run.smithi161.stdout: elfutils-libelf-devel-0.186-1.el8.x86_64 2023-03-27T10:14:28.168 INFO:teuthology.orchestra.run.smithi161.stdout: flex-2.6.1-9.el8.x86_64 2023-03-27T10:14:28.168 INFO:teuthology.orchestra.run.smithi161.stdout: fuse-2.9.7-15.el8.x86_64 2023-03-27T10:14:28.169 INFO:teuthology.orchestra.run.smithi161.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-03-27T10:14:28.169 INFO:teuthology.orchestra.run.smithi161.stdout: jq-1.6-3.el8.x86_64 2023-03-27T10:14:28.169 INFO:teuthology.orchestra.run.smithi161.stdout: keyutils-libs-devel-1.5.10-9.el8.x86_64 2023-03-27T10:14:28.169 INFO:teuthology.orchestra.run.smithi161.stdout: krb5-devel-1.18.2-14.el8.x86_64 2023-03-27T10:14:28.169 INFO:teuthology.orchestra.run.smithi161.stdout: libaio-devel-0.3.112-1.el8.x86_64 2023-03-27T10:14:28.169 INFO:teuthology.orchestra.run.smithi161.stdout: libcephfs-devel-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.169 INFO:teuthology.orchestra.run.smithi161.stdout: libcephfs2-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.170 INFO:teuthology.orchestra.run.smithi161.stdout: libcephsqlite-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.170 INFO:teuthology.orchestra.run.smithi161.stdout: libcom_err-devel-1.45.6-4.el8.x86_64 2023-03-27T10:14:28.170 INFO:teuthology.orchestra.run.smithi161.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-03-27T10:14:28.170 INFO:teuthology.orchestra.run.smithi161.stdout: liboath-2.6.2-3.el8.x86_64 2023-03-27T10:14:28.170 INFO:teuthology.orchestra.run.smithi161.stdout: libpq-13.5-1.el8.x86_64 2023-03-27T10:14:28.170 INFO:teuthology.orchestra.run.smithi161.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-03-27T10:14:28.170 INFO:teuthology.orchestra.run.smithi161.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-03-27T10:14:28.171 INFO:teuthology.orchestra.run.smithi161.stdout: librados-devel-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.171 INFO:teuthology.orchestra.run.smithi161.stdout: libradosstriper1-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.171 INFO:teuthology.orchestra.run.smithi161.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-03-27T10:14:28.171 INFO:teuthology.orchestra.run.smithi161.stdout: librgw2-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.171 INFO:teuthology.orchestra.run.smithi161.stdout: libselinux-devel-2.9-5.el8.x86_64 2023-03-27T10:14:28.171 INFO:teuthology.orchestra.run.smithi161.stdout: libsepol-devel-2.9-3.el8.x86_64 2023-03-27T10:14:28.171 INFO:teuthology.orchestra.run.smithi161.stdout: libtool-2.4.6-25.el8.x86_64 2023-03-27T10:14:28.172 INFO:teuthology.orchestra.run.smithi161.stdout: libunwind-1.3.1-3.el8.x86_64 2023-03-27T10:14:28.172 INFO:teuthology.orchestra.run.smithi161.stdout: libuuid-devel-2.32.1-35.el8.x86_64 2023-03-27T10:14:28.172 INFO:teuthology.orchestra.run.smithi161.stdout: libverto-devel-0.3.0-5.el8.x86_64 2023-03-27T10:14:28.172 INFO:teuthology.orchestra.run.smithi161.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-03-27T10:14:28.172 INFO:teuthology.orchestra.run.smithi161.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-03-27T10:14:28.172 INFO:teuthology.orchestra.run.smithi161.stdout: openblas-0.3.15-3.el8.x86_64 2023-03-27T10:14:28.172 INFO:teuthology.orchestra.run.smithi161.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-03-27T10:14:28.173 INFO:teuthology.orchestra.run.smithi161.stdout: openssl-devel-1:1.1.1k-7.el8_6.x86_64 2023-03-27T10:14:28.173 INFO:teuthology.orchestra.run.smithi161.stdout: pcre2-devel-10.32-2.el8.x86_64 2023-03-27T10:14:28.173 INFO:teuthology.orchestra.run.smithi161.stdout: pcre2-utf16-10.32-2.el8.x86_64 2023-03-27T10:14:28.173 INFO:teuthology.orchestra.run.smithi161.stdout: pcre2-utf32-10.32-2.el8.x86_64 2023-03-27T10:14:28.173 INFO:teuthology.orchestra.run.smithi161.stdout: postgresql-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-03-27T10:14:28.173 INFO:teuthology.orchestra.run.smithi161.stdout: postgresql-contrib-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-03-27T10:14:28.173 INFO:teuthology.orchestra.run.smithi161.stdout: postgresql-server-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-03-27T10:14:28.174 INFO:teuthology.orchestra.run.smithi161.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-03-27T10:14:28.174 INFO:teuthology.orchestra.run.smithi161.stdout: python3-babel-2.5.1-7.el8.noarch 2023-03-27T10:14:28.174 INFO:teuthology.orchestra.run.smithi161.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-03-27T10:14:28.174 INFO:teuthology.orchestra.run.smithi161.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-03-27T10:14:28.174 INFO:teuthology.orchestra.run.smithi161.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-03-27T10:14:28.174 INFO:teuthology.orchestra.run.smithi161.stdout: python3-ceph-argparse-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.174 INFO:teuthology.orchestra.run.smithi161.stdout: python3-ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.174 INFO:teuthology.orchestra.run.smithi161.stdout: python3-cephfs-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.175 INFO:teuthology.orchestra.run.smithi161.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-03-27T10:14:28.175 INFO:teuthology.orchestra.run.smithi161.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-03-27T10:14:28.175 INFO:teuthology.orchestra.run.smithi161.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-03-27T10:14:28.175 INFO:teuthology.orchestra.run.smithi161.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-03-27T10:14:28.175 INFO:teuthology.orchestra.run.smithi161.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-03-27T10:14:28.175 INFO:teuthology.orchestra.run.smithi161.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-03-27T10:14:28.175 INFO:teuthology.orchestra.run.smithi161.stdout: python3-jaraco-6.2-6.el8.noarch 2023-03-27T10:14:28.175 INFO:teuthology.orchestra.run.smithi161.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-03-27T10:14:28.176 INFO:teuthology.orchestra.run.smithi161.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-03-27T10:14:28.176 INFO:teuthology.orchestra.run.smithi161.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-03-27T10:14:28.176 INFO:teuthology.orchestra.run.smithi161.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-03-27T10:14:28.176 INFO:teuthology.orchestra.run.smithi161.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-03-27T10:14:28.176 INFO:teuthology.orchestra.run.smithi161.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-03-27T10:14:28.176 INFO:teuthology.orchestra.run.smithi161.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-03-27T10:14:28.176 INFO:teuthology.orchestra.run.smithi161.stdout: python3-mako-1.0.6-13.el8.noarch 2023-03-27T10:14:28.176 INFO:teuthology.orchestra.run.smithi161.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-03-27T10:14:28.177 INFO:teuthology.orchestra.run.smithi161.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-03-27T10:14:28.177 INFO:teuthology.orchestra.run.smithi161.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-03-27T10:14:28.177 INFO:teuthology.orchestra.run.smithi161.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-03-27T10:14:28.177 INFO:teuthology.orchestra.run.smithi161.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-03-27T10:14:28.177 INFO:teuthology.orchestra.run.smithi161.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-03-27T10:14:28.177 INFO:teuthology.orchestra.run.smithi161.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-03-27T10:14:28.178 INFO:teuthology.orchestra.run.smithi161.stdout: python3-portend-2.6-1.el8.noarch 2023-03-27T10:14:28.178 INFO:teuthology.orchestra.run.smithi161.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-03-27T10:14:28.178 INFO:teuthology.orchestra.run.smithi161.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-03-27T10:14:28.178 INFO:teuthology.orchestra.run.smithi161.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-03-27T10:14:28.178 INFO:teuthology.orchestra.run.smithi161.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-03-27T10:14:28.178 INFO:teuthology.orchestra.run.smithi161.stdout: python3-pycparser-2.14-14.el8.noarch 2023-03-27T10:14:28.178 INFO:teuthology.orchestra.run.smithi161.stdout: python3-pytz-2017.2-9.el8.noarch 2023-03-27T10:14:28.179 INFO:teuthology.orchestra.run.smithi161.stdout: python3-rados-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.179 INFO:teuthology.orchestra.run.smithi161.stdout: python3-rbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.179 INFO:teuthology.orchestra.run.smithi161.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-03-27T10:14:28.179 INFO:teuthology.orchestra.run.smithi161.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-03-27T10:14:28.179 INFO:teuthology.orchestra.run.smithi161.stdout: python3-rgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.179 INFO:teuthology.orchestra.run.smithi161.stdout: python3-routes-2.4.1-12.el8.noarch 2023-03-27T10:14:28.179 INFO:teuthology.orchestra.run.smithi161.stdout: python3-rsa-4.8-1.el8.noarch 2023-03-27T10:14:28.179 INFO:teuthology.orchestra.run.smithi161.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-03-27T10:14:28.180 INFO:teuthology.orchestra.run.smithi161.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-03-27T10:14:28.180 INFO:teuthology.orchestra.run.smithi161.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-03-27T10:14:28.180 INFO:teuthology.orchestra.run.smithi161.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-03-27T10:14:28.180 INFO:teuthology.orchestra.run.smithi161.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-03-27T10:14:28.180 INFO:teuthology.orchestra.run.smithi161.stdout: python3-waitress-1.2.1-2.el8.1.noarch 2023-03-27T10:14:28.181 INFO:teuthology.orchestra.run.smithi161.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-03-27T10:14:28.181 INFO:teuthology.orchestra.run.smithi161.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-03-27T10:14:28.181 INFO:teuthology.orchestra.run.smithi161.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-03-27T10:14:28.181 INFO:teuthology.orchestra.run.smithi161.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-03-27T10:14:28.181 INFO:teuthology.orchestra.run.smithi161.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-03-27T10:14:28.181 INFO:teuthology.orchestra.run.smithi161.stdout: rbd-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.181 INFO:teuthology.orchestra.run.smithi161.stdout: rbd-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.181 INFO:teuthology.orchestra.run.smithi161.stdout: rbd-nbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:28.182 INFO:teuthology.orchestra.run.smithi161.stdout: socat-1.7.4.1-1.el8.x86_64 2023-03-27T10:14:28.182 INFO:teuthology.orchestra.run.smithi161.stdout: thrift-0.13.0-2.el8.x86_64 2023-03-27T10:14:28.182 INFO:teuthology.orchestra.run.smithi161.stdout: uuid-1.6.2-43.el8.x86_64 2023-03-27T10:14:28.182 INFO:teuthology.orchestra.run.smithi161.stdout: xfsprogs-devel-5.0.0-10.el8.x86_64 2023-03-27T10:14:28.183 INFO:teuthology.orchestra.run.smithi161.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-03-27T10:14:28.183 INFO:teuthology.orchestra.run.smithi161.stdout: zlib-devel-1.2.11-18.el8_5.x86_64 2023-03-27T10:14:28.184 INFO:teuthology.orchestra.run.smithi161.stdout: 2023-03-27T10:14:28.184 INFO:teuthology.orchestra.run.smithi161.stdout:Complete! 2023-03-27T10:14:28.383 DEBUG:teuthology.parallel:result is None 2023-03-27T10:14:30.535 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : iproute-5.15.0-4.el8.x86_64 134/134 2023-03-27T10:14:30.536 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:02:01 ago on Mon 27 Mar 2023 10:12:29 AM UTC. 2023-03-27T10:14:33.057 INFO:teuthology.orchestra.run.smithi159.stdout:Installed products updated. 2023-03-27T10:14:33.273 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:14:33.273 INFO:teuthology.orchestra.run.smithi159.stdout:Upgraded: 2023-03-27T10:14:33.273 INFO:teuthology.orchestra.run.smithi159.stdout: iproute-5.15.0-4.el8_6.1.x86_64 2023-03-27T10:14:33.273 INFO:teuthology.orchestra.run.smithi159.stdout: librados2-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.273 INFO:teuthology.orchestra.run.smithi159.stdout: librbd1-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.274 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2023-03-27T10:14:33.274 INFO:teuthology.orchestra.run.smithi159.stdout: NetworkManager-initscripts-updown-1:1.36.0-9.el8_6.noarch 2023-03-27T10:14:33.274 INFO:teuthology.orchestra.run.smithi159.stdout: automake-1.16.1-7.el8.noarch 2023-03-27T10:14:33.274 INFO:teuthology.orchestra.run.smithi159.stdout: bison-3.0.4-10.el8.x86_64 2023-03-27T10:14:33.274 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.274 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-base-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.274 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.274 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.275 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-grafana-dashboards-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:33.275 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-immutable-object-cache-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.275 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mds-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.275 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.275 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-cephadm-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:33.275 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-dashboard-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:33.275 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-diskprediction-local-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:33.276 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-modules-core-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:33.276 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-rook-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:33.276 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mon-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.276 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-osd-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.276 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-prometheus-alerts-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:33.276 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-radosgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.276 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-selinux-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.276 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-test-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.277 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-volume-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:33.277 INFO:teuthology.orchestra.run.smithi159.stdout: cephadm-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:33.277 INFO:teuthology.orchestra.run.smithi159.stdout: cephfs-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.277 INFO:teuthology.orchestra.run.smithi159.stdout: cephfs-top-2:18.0.0-2533.gb70fdf97.el8.noarch 2023-03-27T10:14:33.277 INFO:teuthology.orchestra.run.smithi159.stdout: elfutils-libelf-devel-0.186-1.el8.x86_64 2023-03-27T10:14:33.277 INFO:teuthology.orchestra.run.smithi159.stdout: flex-2.6.1-9.el8.x86_64 2023-03-27T10:14:33.277 INFO:teuthology.orchestra.run.smithi159.stdout: fuse-2.9.7-15.el8.x86_64 2023-03-27T10:14:33.277 INFO:teuthology.orchestra.run.smithi159.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-03-27T10:14:33.278 INFO:teuthology.orchestra.run.smithi159.stdout: jq-1.6-3.el8.x86_64 2023-03-27T10:14:33.278 INFO:teuthology.orchestra.run.smithi159.stdout: keyutils-libs-devel-1.5.10-9.el8.x86_64 2023-03-27T10:14:33.278 INFO:teuthology.orchestra.run.smithi159.stdout: krb5-devel-1.18.2-14.el8.x86_64 2023-03-27T10:14:33.278 INFO:teuthology.orchestra.run.smithi159.stdout: libaio-devel-0.3.112-1.el8.x86_64 2023-03-27T10:14:33.278 INFO:teuthology.orchestra.run.smithi159.stdout: libcephfs-devel-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.278 INFO:teuthology.orchestra.run.smithi159.stdout: libcephfs2-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.278 INFO:teuthology.orchestra.run.smithi159.stdout: libcephsqlite-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.279 INFO:teuthology.orchestra.run.smithi159.stdout: libcom_err-devel-1.45.6-4.el8.x86_64 2023-03-27T10:14:33.279 INFO:teuthology.orchestra.run.smithi159.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-03-27T10:14:33.279 INFO:teuthology.orchestra.run.smithi159.stdout: liboath-2.6.2-3.el8.x86_64 2023-03-27T10:14:33.279 INFO:teuthology.orchestra.run.smithi159.stdout: libpq-13.5-1.el8.x86_64 2023-03-27T10:14:33.279 INFO:teuthology.orchestra.run.smithi159.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-03-27T10:14:33.279 INFO:teuthology.orchestra.run.smithi159.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-03-27T10:14:33.279 INFO:teuthology.orchestra.run.smithi159.stdout: librados-devel-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.279 INFO:teuthology.orchestra.run.smithi159.stdout: libradosstriper1-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.280 INFO:teuthology.orchestra.run.smithi159.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-03-27T10:14:33.280 INFO:teuthology.orchestra.run.smithi159.stdout: librgw2-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.280 INFO:teuthology.orchestra.run.smithi159.stdout: libselinux-devel-2.9-5.el8.x86_64 2023-03-27T10:14:33.280 INFO:teuthology.orchestra.run.smithi159.stdout: libsepol-devel-2.9-3.el8.x86_64 2023-03-27T10:14:33.280 INFO:teuthology.orchestra.run.smithi159.stdout: libtool-2.4.6-25.el8.x86_64 2023-03-27T10:14:33.280 INFO:teuthology.orchestra.run.smithi159.stdout: libunwind-1.3.1-3.el8.x86_64 2023-03-27T10:14:33.280 INFO:teuthology.orchestra.run.smithi159.stdout: libuuid-devel-2.32.1-35.el8.x86_64 2023-03-27T10:14:33.280 INFO:teuthology.orchestra.run.smithi159.stdout: libverto-devel-0.3.0-5.el8.x86_64 2023-03-27T10:14:33.281 INFO:teuthology.orchestra.run.smithi159.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-03-27T10:14:33.281 INFO:teuthology.orchestra.run.smithi159.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-03-27T10:14:33.281 INFO:teuthology.orchestra.run.smithi159.stdout: openblas-0.3.15-3.el8.x86_64 2023-03-27T10:14:33.281 INFO:teuthology.orchestra.run.smithi159.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-03-27T10:14:33.281 INFO:teuthology.orchestra.run.smithi159.stdout: openssl-devel-1:1.1.1k-7.el8_6.x86_64 2023-03-27T10:14:33.281 INFO:teuthology.orchestra.run.smithi159.stdout: pcre2-devel-10.32-2.el8.x86_64 2023-03-27T10:14:33.281 INFO:teuthology.orchestra.run.smithi159.stdout: pcre2-utf16-10.32-2.el8.x86_64 2023-03-27T10:14:33.281 INFO:teuthology.orchestra.run.smithi159.stdout: pcre2-utf32-10.32-2.el8.x86_64 2023-03-27T10:14:33.282 INFO:teuthology.orchestra.run.smithi159.stdout: postgresql-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-03-27T10:14:33.282 INFO:teuthology.orchestra.run.smithi159.stdout: postgresql-contrib-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-03-27T10:14:33.282 INFO:teuthology.orchestra.run.smithi159.stdout: postgresql-server-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-03-27T10:14:33.282 INFO:teuthology.orchestra.run.smithi159.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-03-27T10:14:33.282 INFO:teuthology.orchestra.run.smithi159.stdout: python3-babel-2.5.1-7.el8.noarch 2023-03-27T10:14:33.282 INFO:teuthology.orchestra.run.smithi159.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-03-27T10:14:33.282 INFO:teuthology.orchestra.run.smithi159.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-03-27T10:14:33.283 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-03-27T10:14:33.283 INFO:teuthology.orchestra.run.smithi159.stdout: python3-ceph-argparse-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.283 INFO:teuthology.orchestra.run.smithi159.stdout: python3-ceph-common-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.283 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cephfs-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.283 INFO:teuthology.orchestra.run.smithi159.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-03-27T10:14:33.283 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-03-27T10:14:33.283 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-03-27T10:14:33.283 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-03-27T10:14:33.284 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-03-27T10:14:33.284 INFO:teuthology.orchestra.run.smithi159.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-03-27T10:14:33.284 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jaraco-6.2-6.el8.noarch 2023-03-27T10:14:33.284 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-03-27T10:14:33.284 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-03-27T10:14:33.284 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-03-27T10:14:33.284 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-03-27T10:14:33.284 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-03-27T10:14:33.285 INFO:teuthology.orchestra.run.smithi159.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-03-27T10:14:33.285 INFO:teuthology.orchestra.run.smithi159.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-03-27T10:14:33.285 INFO:teuthology.orchestra.run.smithi159.stdout: python3-mako-1.0.6-13.el8.noarch 2023-03-27T10:14:33.285 INFO:teuthology.orchestra.run.smithi159.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-03-27T10:14:33.285 INFO:teuthology.orchestra.run.smithi159.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-03-27T10:14:33.285 INFO:teuthology.orchestra.run.smithi159.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-03-27T10:14:33.285 INFO:teuthology.orchestra.run.smithi159.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-03-27T10:14:33.285 INFO:teuthology.orchestra.run.smithi159.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-03-27T10:14:33.286 INFO:teuthology.orchestra.run.smithi159.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-03-27T10:14:33.286 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-03-27T10:14:33.286 INFO:teuthology.orchestra.run.smithi159.stdout: python3-portend-2.6-1.el8.noarch 2023-03-27T10:14:33.286 INFO:teuthology.orchestra.run.smithi159.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-03-27T10:14:33.286 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-03-27T10:14:33.286 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-03-27T10:14:33.286 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-03-27T10:14:33.286 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pycparser-2.14-14.el8.noarch 2023-03-27T10:14:33.287 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pytz-2017.2-9.el8.noarch 2023-03-27T10:14:33.287 INFO:teuthology.orchestra.run.smithi159.stdout: python3-rados-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.287 INFO:teuthology.orchestra.run.smithi159.stdout: python3-rbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.287 INFO:teuthology.orchestra.run.smithi159.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-03-27T10:14:33.287 INFO:teuthology.orchestra.run.smithi159.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-03-27T10:14:33.287 INFO:teuthology.orchestra.run.smithi159.stdout: python3-rgw-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.287 INFO:teuthology.orchestra.run.smithi159.stdout: python3-routes-2.4.1-12.el8.noarch 2023-03-27T10:14:33.287 INFO:teuthology.orchestra.run.smithi159.stdout: python3-rsa-4.8-1.el8.noarch 2023-03-27T10:14:33.288 INFO:teuthology.orchestra.run.smithi159.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-03-27T10:14:33.288 INFO:teuthology.orchestra.run.smithi159.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-03-27T10:14:33.288 INFO:teuthology.orchestra.run.smithi159.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-03-27T10:14:33.288 INFO:teuthology.orchestra.run.smithi159.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-03-27T10:14:33.288 INFO:teuthology.orchestra.run.smithi159.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-03-27T10:14:33.288 INFO:teuthology.orchestra.run.smithi159.stdout: python3-waitress-1.2.1-2.el8.1.noarch 2023-03-27T10:14:33.288 INFO:teuthology.orchestra.run.smithi159.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-03-27T10:14:33.289 INFO:teuthology.orchestra.run.smithi159.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-03-27T10:14:33.289 INFO:teuthology.orchestra.run.smithi159.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-03-27T10:14:33.289 INFO:teuthology.orchestra.run.smithi159.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-03-27T10:14:33.289 INFO:teuthology.orchestra.run.smithi159.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-03-27T10:14:33.289 INFO:teuthology.orchestra.run.smithi159.stdout: rbd-fuse-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.289 INFO:teuthology.orchestra.run.smithi159.stdout: rbd-mirror-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.289 INFO:teuthology.orchestra.run.smithi159.stdout: rbd-nbd-2:18.0.0-2533.gb70fdf97.el8.x86_64 2023-03-27T10:14:33.289 INFO:teuthology.orchestra.run.smithi159.stdout: socat-1.7.4.1-1.el8.x86_64 2023-03-27T10:14:33.290 INFO:teuthology.orchestra.run.smithi159.stdout: thrift-0.13.0-2.el8.x86_64 2023-03-27T10:14:33.290 INFO:teuthology.orchestra.run.smithi159.stdout: uuid-1.6.2-43.el8.x86_64 2023-03-27T10:14:33.290 INFO:teuthology.orchestra.run.smithi159.stdout: xfsprogs-devel-5.0.0-10.el8.x86_64 2023-03-27T10:14:33.290 INFO:teuthology.orchestra.run.smithi159.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-03-27T10:14:33.290 INFO:teuthology.orchestra.run.smithi159.stdout: zlib-devel-1.2.11-18.el8_5.x86_64 2023-03-27T10:14:33.291 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:14:33.291 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2023-03-27T10:14:33.490 DEBUG:teuthology.parallel:result is None 2023-03-27T10:14:33.491 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=b70fdf973c2ffa23c0d8aa6918519da05d922302 2023-03-27T10:14:33.630 DEBUG:teuthology.orchestra.run.smithi159:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-03-27T10:14:33.653 INFO:teuthology.orchestra.run.smithi159.stdout:18.0.0-2533.gb70fdf97.el8 2023-03-27T10:14:33.654 INFO:teuthology.packaging:The installed version of ceph is 18.0.0-2533.gb70fdf97.el8 2023-03-27T10:14:33.654 INFO:teuthology.task.install:The correct ceph version 18.0.0-2533.gb70fdf97 is installed. 2023-03-27T10:14:33.655 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=b70fdf973c2ffa23c0d8aa6918519da05d922302 2023-03-27T10:14:33.788 DEBUG:teuthology.orchestra.run.smithi161:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-03-27T10:14:33.816 INFO:teuthology.orchestra.run.smithi161.stdout:18.0.0-2533.gb70fdf97.el8 2023-03-27T10:14:33.817 INFO:teuthology.packaging:The installed version of ceph is 18.0.0-2533.gb70fdf97.el8 2023-03-27T10:14:33.818 INFO:teuthology.task.install:The correct ceph version 18.0.0-2533.gb70fdf97 is installed. 2023-03-27T10:14:33.818 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=b70fdf973c2ffa23c0d8aa6918519da05d922302 2023-03-27T10:14:33.953 DEBUG:teuthology.orchestra.run.smithi171:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-03-27T10:14:33.981 INFO:teuthology.orchestra.run.smithi171.stdout:18.0.0-2533.gb70fdf97.el8 2023-03-27T10:14:33.982 INFO:teuthology.packaging:The installed version of ceph is 18.0.0-2533.gb70fdf97.el8 2023-03-27T10:14:33.982 INFO:teuthology.task.install:The correct ceph version 18.0.0-2533.gb70fdf97 is installed. 2023-03-27T10:14:33.982 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-03-27T10:14:33.983 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2023-03-27T10:14:33.983 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-03-27T10:14:34.025 DEBUG:teuthology.orchestra.run.smithi161:> set -ex 2023-03-27T10:14:34.026 DEBUG:teuthology.orchestra.run.smithi161:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-03-27T10:14:34.073 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2023-03-27T10:14:34.073 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-03-27T10:14:34.120 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-03-27T10:14:34.121 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2023-03-27T10:14:34.121 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/usr/bin/daemon-helper 2023-03-27T10:14:34.149 DEBUG:teuthology.orchestra.run.smithi159:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-03-27T10:14:34.216 DEBUG:teuthology.orchestra.run.smithi161:> set -ex 2023-03-27T10:14:34.217 DEBUG:teuthology.orchestra.run.smithi161:> sudo dd of=/usr/bin/daemon-helper 2023-03-27T10:14:34.248 DEBUG:teuthology.orchestra.run.smithi161:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-03-27T10:14:34.320 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2023-03-27T10:14:34.320 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/usr/bin/daemon-helper 2023-03-27T10:14:34.353 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-03-27T10:14:34.422 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-03-27T10:14:34.422 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2023-03-27T10:14:34.423 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/usr/bin/adjust-ulimits 2023-03-27T10:14:34.450 DEBUG:teuthology.orchestra.run.smithi159:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-03-27T10:14:34.517 DEBUG:teuthology.orchestra.run.smithi161:> set -ex 2023-03-27T10:14:34.517 DEBUG:teuthology.orchestra.run.smithi161:> sudo dd of=/usr/bin/adjust-ulimits 2023-03-27T10:14:34.549 DEBUG:teuthology.orchestra.run.smithi161:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-03-27T10:14:34.620 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2023-03-27T10:14:34.621 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/usr/bin/adjust-ulimits 2023-03-27T10:14:34.651 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-03-27T10:14:34.720 INFO:teuthology.run_tasks:Running task cephadm... 2023-03-27T10:14:34.804 INFO:tasks.cephadm:Config: {'roleless': False, 'cephfs': {'ec_profile': ['m=2', 'k=2', 'crush-failure-domain=osd'], 'max_mds': 5, 'session_timeout': 300, 'standby_replay': True}, 'conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'global': {'ms die on skipped message': False}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 1, 'mds bal fragment size max': 10000, 'mds bal merge size': 5, 'mds bal split bits': 3, 'mds bal split size': 100, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'mds_bal_replicate_threshold': 1, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mgr': {'cephsqlite lock renewal timeout': 900000, 'debug cephsqlite': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon op complaint time': 120}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': 20, 'debug bluestore': 20, 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': 10, 'filestore flush min': 0, 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd heartbeat grace': 60, 'osd objectstore': 'bluestore', 'osd op complaint time': 180, 'osd shutdown pgref assert': True, 'osd_max_omap_entries_per_request': 10}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'overall HEALTH_', '\\(FS_DEGRADED\\)', '\\(MDS_FAILED\\)', '\\(MDS_DEGRADED\\)', '\\(FS_WITH_FAILED_MDS\\)', '\\(MDS_DAMAGE\\)', '\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(FS_INLINE_DATA_DEPRECATED\\)', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'SLOW_OPS', 'slow request'], 'sha1': 'b70fdf973c2ffa23c0d8aa6918519da05d922302', 'subvols': {'create': 2, 'subvol_options': '--namespace-isolated --size 25000000000'}} 2023-03-27T10:14:34.805 INFO:tasks.cephadm:Cluster image is quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 2023-03-27T10:14:34.815 INFO:tasks.cephadm:Cluster fsid is 2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:14:34.816 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-03-27T10:14:34.817 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.159', 'mon.b': '172.21.15.161', 'mon.c': '172.21.15.171'} 2023-03-27T10:14:34.817 INFO:tasks.cephadm:First mon is mon.a on smithi159 2023-03-27T10:14:34.818 INFO:tasks.cephadm:First mgr is x 2023-03-27T10:14:34.818 INFO:tasks.cephadm:Normalizing hostnames... 2023-03-27T10:14:34.819 DEBUG:teuthology.orchestra.run.smithi159:> sudo hostname $(hostname -s) 2023-03-27T10:14:34.849 DEBUG:teuthology.orchestra.run.smithi161:> sudo hostname $(hostname -s) 2023-03-27T10:14:34.883 DEBUG:teuthology.orchestra.run.smithi171:> sudo hostname $(hostname -s) 2023-03-27T10:14:34.913 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra 2023-03-27T10:14:34.914 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=b70fdf973c2ffa23c0d8aa6918519da05d922302 2023-03-27T10:14:35.049 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': 'b70fdf973c2ffa23c0d8aa6918519da05d922302', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-new-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/68740/', 'root_build_cause': 'SCMTRIGGER', 'version': '18.0.0-2533-gb70fdf97', 'node_name': '172.21.2.8+braggi08', 'job_name': 'ceph-dev-new-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic', 'package_manager_version': '18.0.0-2533.gb70fdf97'}, 'url': 'https://1.chacra.ceph.com/r/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/flavors/default/', 'distro_codename': None, 'modified': '2023-03-16 18:35:37.807598', 'distro_version': '8', 'project': 'ceph', 'flavor': 'default', 'ref': 'wip-rishabh-cephman-6', 'chacra_url': 'https://1.chacra.ceph.com/repos/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/flavors/default/', 'archs': ['x86_64', 'source'], 'distro': 'centos'}] 2023-03-27T10:14:35.186 INFO:tasks.util.chacra:got chacra host 1.chacra.ceph.com, ref wip-rishabh-cephman-6, sha1 b70fdf973c2ffa23c0d8aa6918519da05d922302 from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F8%2Fx86_64&flavor=default&sha1=b70fdf973c2ffa23c0d8aa6918519da05d922302 2023-03-27T10:14:35.187 INFO:tasks.cephadm:Discovered cachra url: https://1.chacra.ceph.com/binaries/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/x86_64/flavors/default/cephadm 2023-03-27T10:14:35.187 DEBUG:teuthology.orchestra.run.smithi159:> curl --silent -L https://1.chacra.ceph.com/binaries/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-03-27T10:14:35.546 INFO:teuthology.orchestra.run.smithi159.stdout:-rw-rw-r--. 1 ubuntu ubuntu 378437 Mar 27 10:14 /home/ubuntu/cephtest/cephadm 2023-03-27T10:14:35.546 DEBUG:teuthology.orchestra.run.smithi161:> curl --silent -L https://1.chacra.ceph.com/binaries/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-03-27T10:14:35.889 INFO:teuthology.orchestra.run.smithi161.stdout:-rw-rw-r--. 1 ubuntu ubuntu 378437 Mar 27 10:14 /home/ubuntu/cephtest/cephadm 2023-03-27T10:14:35.890 DEBUG:teuthology.orchestra.run.smithi171:> curl --silent -L https://1.chacra.ceph.com/binaries/ceph/wip-rishabh-cephman-6/b70fdf973c2ffa23c0d8aa6918519da05d922302/centos/8/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-03-27T10:14:36.251 INFO:teuthology.orchestra.run.smithi171.stdout:-rw-rw-r--. 1 ubuntu ubuntu 378437 Mar 27 10:14 /home/ubuntu/cephtest/cephadm 2023-03-27T10:14:36.252 DEBUG:teuthology.orchestra.run.smithi159:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-03-27T10:14:36.273 DEBUG:teuthology.orchestra.run.smithi161:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-03-27T10:14:36.295 DEBUG:teuthology.orchestra.run.smithi171:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-03-27T10:14:36.329 INFO:tasks.cephadm:Pulling image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 on all hosts... 2023-03-27T10:14:36.330 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 pull 2023-03-27T10:14:36.332 DEBUG:teuthology.orchestra.run.smithi161:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 pull 2023-03-27T10:14:36.338 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 pull 2023-03-27T10:14:41.298 INFO:teuthology.orchestra.run.smithi171.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302... 2023-03-27T10:14:41.306 INFO:teuthology.orchestra.run.smithi161.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302... 2023-03-27T10:14:42.389 INFO:teuthology.orchestra.run.smithi159.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302... 2023-03-27T10:15:08.358 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2023-03-27T10:15:08.359 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph_version": "ceph version 18.0.0-2533-gb70fdf97 (b70fdf973c2ffa23c0d8aa6918519da05d922302) reef (dev)", 2023-03-27T10:15:08.359 INFO:teuthology.orchestra.run.smithi159.stdout: "image_id": "f33a0a08478e7117f85687bae78eb2d43228feb0f6fd37944258368393b59e81", 2023-03-27T10:15:08.359 INFO:teuthology.orchestra.run.smithi159.stdout: "repo_digests": [ 2023-03-27T10:15:08.359 INFO:teuthology.orchestra.run.smithi159.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:7c7e2d1649b6a308f519a9b6f8d56adb79ec9d10f2d841463bf796d25761cead" 2023-03-27T10:15:08.359 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2023-03-27T10:15:08.359 INFO:teuthology.orchestra.run.smithi159.stdout:} 2023-03-27T10:15:24.756 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2023-03-27T10:15:24.756 INFO:teuthology.orchestra.run.smithi171.stdout: "ceph_version": "ceph version 18.0.0-2533-gb70fdf97 (b70fdf973c2ffa23c0d8aa6918519da05d922302) reef (dev)", 2023-03-27T10:15:24.757 INFO:teuthology.orchestra.run.smithi171.stdout: "image_id": "f33a0a08478e7117f85687bae78eb2d43228feb0f6fd37944258368393b59e81", 2023-03-27T10:15:24.757 INFO:teuthology.orchestra.run.smithi171.stdout: "repo_digests": [ 2023-03-27T10:15:24.757 INFO:teuthology.orchestra.run.smithi171.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:7c7e2d1649b6a308f519a9b6f8d56adb79ec9d10f2d841463bf796d25761cead" 2023-03-27T10:15:24.757 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2023-03-27T10:15:24.757 INFO:teuthology.orchestra.run.smithi171.stdout:} 2023-03-27T10:15:24.779 INFO:teuthology.orchestra.run.smithi161.stdout:{ 2023-03-27T10:15:24.780 INFO:teuthology.orchestra.run.smithi161.stdout: "ceph_version": "ceph version 18.0.0-2533-gb70fdf97 (b70fdf973c2ffa23c0d8aa6918519da05d922302) reef (dev)", 2023-03-27T10:15:24.780 INFO:teuthology.orchestra.run.smithi161.stdout: "image_id": "f33a0a08478e7117f85687bae78eb2d43228feb0f6fd37944258368393b59e81", 2023-03-27T10:15:24.780 INFO:teuthology.orchestra.run.smithi161.stdout: "repo_digests": [ 2023-03-27T10:15:24.780 INFO:teuthology.orchestra.run.smithi161.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:7c7e2d1649b6a308f519a9b6f8d56adb79ec9d10f2d841463bf796d25761cead" 2023-03-27T10:15:24.780 INFO:teuthology.orchestra.run.smithi161.stdout: ] 2023-03-27T10:15:24.781 INFO:teuthology.orchestra.run.smithi161.stdout:} 2023-03-27T10:15:24.801 DEBUG:teuthology.orchestra.run.smithi159:> sudo mkdir -p /etc/ceph 2023-03-27T10:15:24.840 DEBUG:teuthology.orchestra.run.smithi161:> sudo mkdir -p /etc/ceph 2023-03-27T10:15:24.870 DEBUG:teuthology.orchestra.run.smithi171:> sudo mkdir -p /etc/ceph 2023-03-27T10:15:24.901 DEBUG:teuthology.orchestra.run.smithi159:> sudo chmod 777 /etc/ceph 2023-03-27T10:15:24.932 DEBUG:teuthology.orchestra.run.smithi161:> sudo chmod 777 /etc/ceph 2023-03-27T10:15:24.957 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod 777 /etc/ceph 2023-03-27T10:15:24.983 INFO:tasks.cephadm:Writing seed config... 2023-03-27T10:15:24.984 INFO:tasks.cephadm: override: [client] client mount timeout = 600 2023-03-27T10:15:24.985 INFO:tasks.cephadm: override: [client] debug client = 20 2023-03-27T10:15:24.985 INFO:tasks.cephadm: override: [client] debug ms = 1 2023-03-27T10:15:24.985 INFO:tasks.cephadm: override: [client] rados mon op timeout = 900 2023-03-27T10:15:24.985 INFO:tasks.cephadm: override: [client] rados osd op timeout = 900 2023-03-27T10:15:24.985 INFO:tasks.cephadm: override: [global] ms die on skipped message = False 2023-03-27T10:15:24.986 INFO:tasks.cephadm: override: [mds] debug mds = 20 2023-03-27T10:15:24.986 INFO:tasks.cephadm: override: [mds] debug mds balancer = 20 2023-03-27T10:15:24.986 INFO:tasks.cephadm: override: [mds] debug ms = 1 2023-03-27T10:15:24.986 INFO:tasks.cephadm: override: [mds] mds bal fragment size max = 10000 2023-03-27T10:15:24.986 INFO:tasks.cephadm: override: [mds] mds bal merge size = 5 2023-03-27T10:15:24.987 INFO:tasks.cephadm: override: [mds] mds bal split bits = 3 2023-03-27T10:15:24.987 INFO:tasks.cephadm: override: [mds] mds bal split size = 100 2023-03-27T10:15:24.987 INFO:tasks.cephadm: override: [mds] mds debug frag = True 2023-03-27T10:15:24.987 INFO:tasks.cephadm: override: [mds] mds debug scatterstat = True 2023-03-27T10:15:24.988 INFO:tasks.cephadm: override: [mds] mds op complaint time = 180 2023-03-27T10:15:24.988 INFO:tasks.cephadm: override: [mds] mds verify scatter = True 2023-03-27T10:15:24.988 INFO:tasks.cephadm: override: [mds] mds_bal_replicate_threshold = 1 2023-03-27T10:15:24.988 INFO:tasks.cephadm: override: [mds] osd op complaint time = 180 2023-03-27T10:15:24.988 INFO:tasks.cephadm: override: [mds] rados mon op timeout = 900 2023-03-27T10:15:24.989 INFO:tasks.cephadm: override: [mds] rados osd op timeout = 900 2023-03-27T10:15:24.989 INFO:tasks.cephadm: override: [mgr] cephsqlite lock renewal timeout = 900000 2023-03-27T10:15:24.989 INFO:tasks.cephadm: override: [mgr] debug cephsqlite = 20 2023-03-27T10:15:24.989 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-03-27T10:15:24.989 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-03-27T10:15:24.990 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-03-27T10:15:24.990 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-03-27T10:15:24.990 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-03-27T10:15:24.990 INFO:tasks.cephadm: override: [mon] mon op complaint time = 120 2023-03-27T10:15:24.990 INFO:tasks.cephadm: override: [osd] bluestore block size = 96636764160 2023-03-27T10:15:24.991 INFO:tasks.cephadm: override: [osd] bluestore compression mode = aggressive 2023-03-27T10:15:24.991 INFO:tasks.cephadm: override: [osd] bluestore fsck on mount = True 2023-03-27T10:15:24.991 INFO:tasks.cephadm: override: [osd] debug bluefs = 20 2023-03-27T10:15:24.991 INFO:tasks.cephadm: override: [osd] debug bluestore = 20 2023-03-27T10:15:24.991 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-03-27T10:15:24.992 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-03-27T10:15:24.992 INFO:tasks.cephadm: override: [osd] debug rocksdb = 10 2023-03-27T10:15:24.992 INFO:tasks.cephadm: override: [osd] filestore flush min = 0 2023-03-27T10:15:24.992 INFO:tasks.cephadm: override: [osd] mon osd backfillfull_ratio = 0.85 2023-03-27T10:15:24.992 INFO:tasks.cephadm: override: [osd] mon osd full ratio = 0.9 2023-03-27T10:15:24.993 INFO:tasks.cephadm: override: [osd] mon osd nearfull ratio = 0.8 2023-03-27T10:15:24.993 INFO:tasks.cephadm: override: [osd] osd failsafe full ratio = 0.95 2023-03-27T10:15:24.993 INFO:tasks.cephadm: override: [osd] osd heartbeat grace = 60 2023-03-27T10:15:24.993 INFO:tasks.cephadm: override: [osd] osd objectstore = bluestore 2023-03-27T10:15:24.993 INFO:tasks.cephadm: override: [osd] osd op complaint time = 180 2023-03-27T10:15:24.994 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-03-27T10:15:24.994 INFO:tasks.cephadm: override: [osd] osd_max_omap_entries_per_request = 10 2023-03-27T10:15:24.995 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2023-03-27T10:15:24.995 DEBUG:teuthology.orchestra.run.smithi159:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-03-27T10:15:25.013 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 2c3cd140-cc88-11ed-9afe-001a4aab830c ms die on skipped message = False [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true bluestore block size = 96636764160 bluestore compression mode = aggressive bluestore fsck on mount = True debug bluefs = 20 debug bluestore = 20 debug ms = 1 debug osd = 20 debug rocksdb = 10 filestore flush min = 0 mon osd backfillfull_ratio = 0.85 mon osd full ratio = 0.9 mon osd nearfull ratio = 0.8 osd failsafe full ratio = 0.95 osd heartbeat grace = 60 osd objectstore = bluestore osd op complaint time = 180 osd_max_omap_entries_per_request = 10 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false cephsqlite lock renewal timeout = 900000 debug cephsqlite = 20 debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 mon op complaint time = 120 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true [client] client mount timeout = 600 debug client = 20 debug ms = 1 rados mon op timeout = 900 rados osd op timeout = 900 [mds] debug mds = 20 debug mds balancer = 20 debug ms = 1 mds bal fragment size max = 10000 mds bal merge size = 5 mds bal split bits = 3 mds bal split size = 100 mds debug frag = True mds debug scatterstat = True mds op complaint time = 180 mds verify scatter = True mds_bal_replicate_threshold = 1 osd op complaint time = 180 rados mon op timeout = 900 rados osd op timeout = 900 2023-03-27T10:15:25.014 DEBUG:teuthology.orchestra.run.smithi159:mon.a> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mon.a.service 2023-03-27T10:15:25.055 DEBUG:teuthology.orchestra.run.smithi159:mgr.x> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mgr.x.service 2023-03-27T10:15:25.084 INFO:journalctl@ceph.mon.a.smithi159.stdout:-- Logs begin at Mon 2023-03-27 10:03:12 UTC. -- 2023-03-27T10:15:25.085 INFO:tasks.cephadm:Bootstrapping... 2023-03-27T10:15:25.086 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 -v bootstrap --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id x --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.159 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-03-27T10:15:25.112 INFO:journalctl@ceph.mgr.x.smithi159.stdout:-- Logs begin at Mon 2023-03-27 10:03:12 UTC. -- 2023-03-27T10:15:25.335 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2023-03-27T10:15:25.335 INFO:teuthology.orchestra.run.smithi159.stdout:cephadm ['--image', 'quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302', '-v', 'bootstrap', '--fsid', '2c3cd140-cc88-11ed-9afe-001a4aab830c', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'x', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.159', '--skip-admin-label'] 2023-03-27T10:15:25.485 INFO:teuthology.orchestra.run.smithi159.stdout:/bin/podman: stdout 4.1.1 2023-03-27T10:15:25.486 INFO:teuthology.orchestra.run.smithi159.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-03-27T10:15:25.486 INFO:teuthology.orchestra.run.smithi159.stdout:Verifying podman|docker is present... 2023-03-27T10:15:25.627 INFO:teuthology.orchestra.run.smithi159.stdout:/bin/podman: stdout 4.1.1 2023-03-27T10:15:25.628 INFO:teuthology.orchestra.run.smithi159.stdout:Verifying lvm2 is present... 2023-03-27T10:15:25.628 INFO:teuthology.orchestra.run.smithi159.stdout:Verifying time synchronization is in place... 2023-03-27T10:15:25.635 INFO:teuthology.orchestra.run.smithi159.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-03-27T10:15:25.635 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-03-27T10:15:25.640 INFO:teuthology.orchestra.run.smithi159.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-03-27T10:15:25.641 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stdout inactive 2023-03-27T10:15:25.647 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stdout enabled 2023-03-27T10:15:25.653 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stdout active 2023-03-27T10:15:25.653 INFO:teuthology.orchestra.run.smithi159.stdout:Unit chronyd.service is enabled and running 2023-03-27T10:15:25.653 INFO:teuthology.orchestra.run.smithi159.stdout:Repeating the final host check... 2023-03-27T10:15:25.794 INFO:teuthology.orchestra.run.smithi159.stdout:/bin/podman: stdout 4.1.1 2023-03-27T10:15:25.795 INFO:teuthology.orchestra.run.smithi159.stdout:podman (/bin/podman) version 4.1.1 is present 2023-03-27T10:15:25.795 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl is present 2023-03-27T10:15:25.795 INFO:teuthology.orchestra.run.smithi159.stdout:lvcreate is present 2023-03-27T10:15:25.801 INFO:teuthology.orchestra.run.smithi159.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-03-27T10:15:25.801 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-03-27T10:15:25.806 INFO:teuthology.orchestra.run.smithi159.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-03-27T10:15:25.806 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stdout inactive 2023-03-27T10:15:25.812 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stdout enabled 2023-03-27T10:15:25.818 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stdout active 2023-03-27T10:15:25.818 INFO:teuthology.orchestra.run.smithi159.stdout:Unit chronyd.service is enabled and running 2023-03-27T10:15:25.818 INFO:teuthology.orchestra.run.smithi159.stdout:Host looks OK 2023-03-27T10:15:25.818 INFO:teuthology.orchestra.run.smithi159.stdout:Cluster fsid: 2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:15:25.819 INFO:teuthology.orchestra.run.smithi159.stdout:Acquiring lock 139631477619176 on /run/cephadm/2c3cd140-cc88-11ed-9afe-001a4aab830c.lock 2023-03-27T10:15:25.819 INFO:teuthology.orchestra.run.smithi159.stdout:Lock 139631477619176 acquired on /run/cephadm/2c3cd140-cc88-11ed-9afe-001a4aab830c.lock 2023-03-27T10:15:25.819 INFO:teuthology.orchestra.run.smithi159.stdout:Verifying IP 172.21.15.159 port 3300 ... 2023-03-27T10:15:25.819 INFO:teuthology.orchestra.run.smithi159.stdout:Verifying IP 172.21.15.159 port 6789 ... 2023-03-27T10:15:25.819 INFO:teuthology.orchestra.run.smithi159.stdout:Base mon IP(s) is [172.21.15.159:3300, 172.21.15.159:6789], mon addrv is [v2:172.21.15.159:3300,v1:172.21.15.159:6789] 2023-03-27T10:15:25.822 INFO:teuthology.orchestra.run.smithi159.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp metric 100 2023-03-27T10:15:25.823 INFO:teuthology.orchestra.run.smithi159.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.159 metric 100 2023-03-27T10:15:25.825 INFO:teuthology.orchestra.run.smithi159.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-03-27T10:15:25.825 INFO:teuthology.orchestra.run.smithi159.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2023-03-27T10:15:25.826 INFO:teuthology.orchestra.run.smithi159.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2023-03-27T10:15:25.828 INFO:teuthology.orchestra.run.smithi159.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-03-27T10:15:25.828 INFO:teuthology.orchestra.run.smithi159.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-03-27T10:15:25.828 INFO:teuthology.orchestra.run.smithi159.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-03-27T10:15:25.828 INFO:teuthology.orchestra.run.smithi159.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-03-27T10:15:25.828 INFO:teuthology.orchestra.run.smithi159.stdout:/sbin/ip: stdout inet6 fe80::967c:9ee3:3b7f:68a0/64 scope link noprefixroute 2023-03-27T10:15:25.828 INFO:teuthology.orchestra.run.smithi159.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-03-27T10:15:25.829 INFO:teuthology.orchestra.run.smithi159.stdout:Mon IP `172.21.15.159` is in CIDR network `172.21.0.0/20` 2023-03-27T10:15:25.829 INFO:teuthology.orchestra.run.smithi159.stdout:Mon IP `172.21.15.159` is in CIDR network `172.21.0.0/20` 2023-03-27T10:15:25.829 INFO:teuthology.orchestra.run.smithi159.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-03-27T10:15:25.830 INFO:teuthology.orchestra.run.smithi159.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-03-27T10:15:25.830 INFO:teuthology.orchestra.run.smithi159.stdout:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302... 2023-03-27T10:15:26.274 INFO:teuthology.orchestra.run.smithi159.stdout:/bin/podman: stdout f33a0a08478e7117f85687bae78eb2d43228feb0f6fd37944258368393b59e81 2023-03-27T10:15:26.274 INFO:teuthology.orchestra.run.smithi159.stdout:/bin/podman: stderr Trying to pull quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302... 2023-03-27T10:15:26.274 INFO:teuthology.orchestra.run.smithi159.stdout:/bin/podman: stderr Getting image source signatures 2023-03-27T10:15:26.274 INFO:teuthology.orchestra.run.smithi159.stdout:/bin/podman: stderr Copying blob sha256:fe8731f29b1640a515314adb1dcb6d38ab56a2b6c924294c0e9610c449e045a7 2023-03-27T10:15:26.274 INFO:teuthology.orchestra.run.smithi159.stdout:/bin/podman: stderr Copying blob sha256:e8273662897a8d884c6f0dc47291edd7e6000e44d20afd185182e3ace70cf9b9 2023-03-27T10:15:26.275 INFO:teuthology.orchestra.run.smithi159.stdout:/bin/podman: stderr Copying config sha256:f33a0a08478e7117f85687bae78eb2d43228feb0f6fd37944258368393b59e81 2023-03-27T10:15:26.275 INFO:teuthology.orchestra.run.smithi159.stdout:/bin/podman: stderr Writing manifest to image destination 2023-03-27T10:15:26.275 INFO:teuthology.orchestra.run.smithi159.stdout:/bin/podman: stderr Storing signatures 2023-03-27T10:15:27.122 INFO:teuthology.orchestra.run.smithi159.stdout:ceph: stdout ceph version 18.0.0-2533-gb70fdf97 (b70fdf973c2ffa23c0d8aa6918519da05d922302) reef (dev) 2023-03-27T10:15:27.122 INFO:teuthology.orchestra.run.smithi159.stdout:Ceph version: ceph version 18.0.0-2533-gb70fdf97 (b70fdf973c2ffa23c0d8aa6918519da05d922302) reef (dev) 2023-03-27T10:15:27.122 INFO:teuthology.orchestra.run.smithi159.stdout:Extracting ceph user uid/gid from container image... 2023-03-27T10:15:27.875 INFO:teuthology.orchestra.run.smithi159.stdout:stat: stdout 167 167 2023-03-27T10:15:27.876 INFO:teuthology.orchestra.run.smithi159.stdout:Creating initial keys... 2023-03-27T10:15:28.619 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph-authtool: stdout AQDAbCFkA66yDhAAcdeN8BxW1QvtPD+GxcyRNQ== 2023-03-27T10:15:29.366 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph-authtool: stdout AQDBbCFkbCKOABAAy2iyGphGjvwNZDnUeSIqQA== 2023-03-27T10:15:30.113 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph-authtool: stdout AQDBbCFk62k6LBAA+Xu85vZ8uo3WG29sFtnuyA== 2023-03-27T10:15:30.114 INFO:teuthology.orchestra.run.smithi159.stdout:Creating initial monmap... 2023-03-27T10:15:30.863 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-03-27T10:15:30.863 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2023-03-27T10:15:30.863 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:15:30.863 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-03-27T10:15:30.863 INFO:teuthology.orchestra.run.smithi159.stdout:monmaptool for a [v2:172.21.15.159:3300,v1:172.21.15.159:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-03-27T10:15:30.864 INFO:teuthology.orchestra.run.smithi159.stdout:setting min_mon_release = pacific 2023-03-27T10:15:30.864 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/monmaptool: set fsid to 2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:15:30.864 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-03-27T10:15:30.864 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:15:30.864 INFO:teuthology.orchestra.run.smithi159.stdout:Creating mon... 2023-03-27T10:15:31.609 INFO:teuthology.orchestra.run.smithi159.stdout:create mon.a on 2023-03-27T10:15:31.704 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-03-27T10:15:31.791 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-03-27T10:15:31.881 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c.target → /etc/systemd/system/ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c.target. 2023-03-27T10:15:31.881 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c.target → /etc/systemd/system/ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c.target. 2023-03-27T10:15:32.007 INFO:teuthology.orchestra.run.smithi159.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mon.a 2023-03-27T10:15:32.008 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stderr Failed to reset failed state of unit ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mon.a.service: Unit ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mon.a.service not loaded. 2023-03-27T10:15:32.094 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c.target.wants/ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mon.a.service → /etc/systemd/system/ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@.service. 2023-03-27T10:15:32.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:32 smithi159 systemd[1]: Starting Ceph mon.a for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:15:32.689 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:32 smithi159 podman[92501]: 2023-03-27T10:15:32.689 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:32 smithi159 bash[92501]: b93ba6714ac4323bf2b7560da80fb2e250b962a324283000609236426fdc1b10 2023-03-27T10:15:32.689 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:32 smithi159 ceph-mon[92542]: mkfs 2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:15:32.689 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:32 smithi159 ceph-mon[92542]: mon.a is new leader, mons a in quorum (ranks 0) 2023-03-27T10:15:32.700 INFO:teuthology.orchestra.run.smithi159.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-03-27T10:15:32.700 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stdout disabled 2023-03-27T10:15:32.707 INFO:teuthology.orchestra.run.smithi159.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-03-27T10:15:32.707 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stdout inactive 2023-03-27T10:15:32.708 INFO:teuthology.orchestra.run.smithi159.stdout:firewalld.service is not enabled 2023-03-27T10:15:32.708 INFO:teuthology.orchestra.run.smithi159.stdout:Not possible to enable service . firewalld.service is not available 2023-03-27T10:15:32.708 INFO:teuthology.orchestra.run.smithi159.stdout:Waiting for mon to start... 2023-03-27T10:15:32.708 INFO:teuthology.orchestra.run.smithi159.stdout:Waiting for mon... 2023-03-27T10:15:32.947 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:32 smithi159 systemd[1]: Started Ceph mon.a for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:15:33.657 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout cluster: 2023-03-27T10:15:33.658 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout id: 2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:15:33.658 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-03-27T10:15:33.658 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 2023-03-27T10:15:33.658 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout services: 2023-03-27T10:15:33.658 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.657587s) 2023-03-27T10:15:33.658 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-03-27T10:15:33.659 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-03-27T10:15:33.659 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 2023-03-27T10:15:33.659 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout data: 2023-03-27T10:15:33.659 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-03-27T10:15:33.659 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-03-27T10:15:33.659 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-03-27T10:15:33.659 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout pgs: 2023-03-27T10:15:33.659 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 2023-03-27T10:15:33.660 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.196+0000 7fc46cf52700 1 Processor -- start 2023-03-27T10:15:33.660 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.196+0000 7fc46cf52700 1 -- start start 2023-03-27T10:15:33.660 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.196+0000 7fc46cf52700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc468104550 0x7fc468104960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:33.660 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.196+0000 7fc46cf52700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc468104ea0 con 0x7fc468104550 2023-03-27T10:15:33.660 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.196+0000 7fc46659c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc468104550 0x7fc468104960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:33.660 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.196+0000 7fc46659c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc468104550 0x7fc468104960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44322/0 (socket says 172.21.15.159:44322) 2023-03-27T10:15:33.660 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.196+0000 7fc46659c700 1 -- 172.21.15.159:0/1277546630 learned_addr learned my addr 172.21.15.159:0/1277546630 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:33.661 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.197+0000 7fc46659c700 1 -- 172.21.15.159:0/1277546630 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc468104fe0 con 0x7fc468104550 2023-03-27T10:15:33.661 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.197+0000 7fc46659c700 1 --2- 172.21.15.159:0/1277546630 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc468104550 0x7fc468104960 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7fc4580098a0 tx=0x7fc458009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dc61bae736e7eff7 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:33.661 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.197+0000 7fc46559a700 1 -- 172.21.15.159:0/1277546630 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fc458004160 con 0x7fc468104550 2023-03-27T10:15:33.661 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.197+0000 7fc46559a700 1 -- 172.21.15.159:0/1277546630 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc4580042c0 con 0x7fc468104550 2023-03-27T10:15:33.661 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.197+0000 7fc46559a700 1 -- 172.21.15.159:0/1277546630 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fc4580045a0 con 0x7fc468104550 2023-03-27T10:15:33.661 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.198+0000 7fc46cf52700 1 -- 172.21.15.159:0/1277546630 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc468104550 msgr2=0x7fc468104960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:33.661 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.198+0000 7fc46cf52700 1 --2- 172.21.15.159:0/1277546630 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc468104550 0x7fc468104960 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7fc4580098a0 tx=0x7fc458009bb0 comp rx=0 tx=0).stop 2023-03-27T10:15:33.662 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.198+0000 7fc46cf52700 1 -- 172.21.15.159:0/1277546630 shutdown_connections 2023-03-27T10:15:33.662 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.198+0000 7fc46cf52700 1 --2- 172.21.15.159:0/1277546630 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc468104550 0x7fc468104960 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:33.662 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.198+0000 7fc46cf52700 1 -- 172.21.15.159:0/1277546630 >> 172.21.15.159:0/1277546630 conn(0x7fc4680ffba0 msgr2=0x7fc468101fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:33.662 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.198+0000 7fc46cf52700 1 -- 172.21.15.159:0/1277546630 shutdown_connections 2023-03-27T10:15:33.662 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.198+0000 7fc46cf52700 1 -- 172.21.15.159:0/1277546630 wait complete. 2023-03-27T10:15:33.662 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.198+0000 7fc46cf52700 1 Processor -- start 2023-03-27T10:15:33.662 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.199+0000 7fc46cf52700 1 -- start start 2023-03-27T10:15:33.663 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.199+0000 7fc46cf52700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc468104550 0x7fc468197a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:33.663 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.199+0000 7fc46cf52700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc468197f50 con 0x7fc468104550 2023-03-27T10:15:33.663 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.199+0000 7fc46659c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc468104550 0x7fc468197a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:33.663 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.199+0000 7fc46659c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc468104550 0x7fc468197a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44324/0 (socket says 172.21.15.159:44324) 2023-03-27T10:15:33.663 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.199+0000 7fc46659c700 1 -- 172.21.15.159:0/862975740 learned_addr learned my addr 172.21.15.159:0/862975740 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:33.663 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.199+0000 7fc46659c700 1 -- 172.21.15.159:0/862975740 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc458009550 con 0x7fc468104550 2023-03-27T10:15:33.663 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.199+0000 7fc46659c700 1 --2- 172.21.15.159:0/862975740 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc468104550 0x7fc468197a10 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7fc468105280 tx=0x7fc458004750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:33.663 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.199+0000 7fc4577fe700 1 -- 172.21.15.159:0/862975740 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fc458011630 con 0x7fc468104550 2023-03-27T10:15:33.664 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.199+0000 7fc4577fe700 1 -- 172.21.15.159:0/862975740 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fc458011790 con 0x7fc468104550 2023-03-27T10:15:33.664 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.199+0000 7fc46cf52700 1 -- 172.21.15.159:0/862975740 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc468105aa0 con 0x7fc468104550 2023-03-27T10:15:33.664 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.199+0000 7fc4577fe700 1 -- 172.21.15.159:0/862975740 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fc458011a30 con 0x7fc468104550 2023-03-27T10:15:33.664 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.199+0000 7fc46cf52700 1 -- 172.21.15.159:0/862975740 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc468105910 con 0x7fc468104550 2023-03-27T10:15:33.664 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.200+0000 7fc4577fe700 1 -- 172.21.15.159:0/862975740 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 1) v1 ==== 656+0+0 (secure 0 0 0) 0x7fc458017020 con 0x7fc468104550 2023-03-27T10:15:33.664 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.200+0000 7fc4577fe700 1 -- 172.21.15.159:0/862975740 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7fc45801c450 con 0x7fc468104550 2023-03-27T10:15:33.665 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.201+0000 7fc46cf52700 1 -- 172.21.15.159:0/862975740 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc468105c30 con 0x7fc468104550 2023-03-27T10:15:33.665 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.203+0000 7fc4577fe700 1 -- 172.21.15.159:0/862975740 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72400 (secure 0 0 0) 0x7fc45801b450 con 0x7fc468104550 2023-03-27T10:15:33.665 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.272+0000 7fc46cf52700 1 -- 172.21.15.159:0/862975740 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "status"} v 0) v1 -- 0x7fc4681985d0 con 0x7fc468104550 2023-03-27T10:15:33.665 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.272+0000 7fc4577fe700 1 -- 172.21.15.159:0/862975740 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "status"}]=0 v0) v1 ==== 54+0+317 (secure 0 0 0) 0x7fc45801b5f0 con 0x7fc468104550 2023-03-27T10:15:33.665 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.273+0000 7fc46cf52700 1 -- 172.21.15.159:0/862975740 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc468104550 msgr2=0x7fc468197a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:33.665 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.273+0000 7fc46cf52700 1 --2- 172.21.15.159:0/862975740 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc468104550 0x7fc468197a10 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7fc468105280 tx=0x7fc458004750 comp rx=0 tx=0).stop 2023-03-27T10:15:33.665 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.273+0000 7fc46cf52700 1 -- 172.21.15.159:0/862975740 shutdown_connections 2023-03-27T10:15:33.666 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.273+0000 7fc46cf52700 1 --2- 172.21.15.159:0/862975740 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc468104550 0x7fc468197a10 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:33.666 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.273+0000 7fc46cf52700 1 -- 172.21.15.159:0/862975740 >> 172.21.15.159:0/862975740 conn(0x7fc4680ffba0 msgr2=0x7fc46818bf90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:33.666 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.273+0000 7fc46cf52700 1 -- 172.21.15.159:0/862975740 shutdown_connections 2023-03-27T10:15:33.666 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:33.273+0000 7fc46cf52700 1 -- 172.21.15.159:0/862975740 wait complete. 2023-03-27T10:15:33.666 INFO:teuthology.orchestra.run.smithi159.stdout:mon is available 2023-03-27T10:15:33.666 INFO:teuthology.orchestra.run.smithi159.stdout:Assimilating anything we can from ceph.conf... 2023-03-27T10:15:33.874 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:33 smithi159 ceph-mon[92542]: mon.a is new leader, mons a in quorum (ranks 0) 2023-03-27T10:15:33.874 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:33 smithi159 ceph-mon[92542]: monmap e1: 1 mons at {a=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0]} removed_ranks: {} 2023-03-27T10:15:33.874 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:33 smithi159 ceph-mon[92542]: fsmap 2023-03-27T10:15:33.875 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:33 smithi159 ceph-mon[92542]: osdmap e1: 0 total, 0 up, 0 in 2023-03-27T10:15:33.875 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:33 smithi159 ceph-mon[92542]: mgrmap e1: no daemons active 2023-03-27T10:15:33.875 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:33 smithi159 ceph-mon[92542]: from='client.? 172.21.15.159:0/862975740' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-03-27T10:15:34.600 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 2023-03-27T10:15:34.601 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout [global] 2023-03-27T10:15:34.601 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout fsid = 2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:15:34.601 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.159:3300,v1:172.21.15.159:6789] 2023-03-27T10:15:34.601 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-03-27T10:15:34.601 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-03-27T10:15:34.601 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-03-27T10:15:34.602 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-03-27T10:15:34.602 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 2023-03-27T10:15:34.602 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout [mgr] 2023-03-27T10:15:34.602 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-03-27T10:15:34.602 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 2023-03-27T10:15:34.602 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout [osd] 2023-03-27T10:15:34.602 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout filestore_flush_min = 0 2023-03-27T10:15:34.603 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mon_osd_backfillfull_ratio = 0.85 2023-03-27T10:15:34.603 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mon_osd_full_ratio = 0.9 2023-03-27T10:15:34.603 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mon_osd_nearfull_ratio = 0.8 2023-03-27T10:15:34.603 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-03-27T10:15:34.603 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-03-27T10:15:34.603 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.095+0000 7f1329d23700 1 Processor -- start 2023-03-27T10:15:34.603 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.095+0000 7f1329d23700 1 -- start start 2023-03-27T10:15:34.603 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.095+0000 7f1329d23700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1324073ba0 0x7f132410a610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:34.604 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.095+0000 7f1329d23700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1324106000 con 0x7f1324073ba0 2023-03-27T10:15:34.604 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.095+0000 7f13237fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1324073ba0 0x7f132410a610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:34.604 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.096+0000 7f13237fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1324073ba0 0x7f132410a610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44326/0 (socket says 172.21.15.159:44326) 2023-03-27T10:15:34.604 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.096+0000 7f13237fe700 1 -- 172.21.15.159:0/197395940 learned_addr learned my addr 172.21.15.159:0/197395940 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:34.604 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.096+0000 7f13237fe700 1 -- 172.21.15.159:0/197395940 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f132410ab50 con 0x7f1324073ba0 2023-03-27T10:15:34.604 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.096+0000 7f13237fe700 1 --2- 172.21.15.159:0/197395940 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1324073ba0 0x7f132410a610 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7f130c0098a0 tx=0x7f130c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5bc961559506899c server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:34.604 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.096+0000 7f13227fc700 1 -- 172.21.15.159:0/197395940 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f130c004160 con 0x7f1324073ba0 2023-03-27T10:15:34.605 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.096+0000 7f13227fc700 1 -- 172.21.15.159:0/197395940 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f130c0042c0 con 0x7f1324073ba0 2023-03-27T10:15:34.605 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.096+0000 7f13227fc700 1 -- 172.21.15.159:0/197395940 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f130c0045a0 con 0x7f1324073ba0 2023-03-27T10:15:34.605 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.097+0000 7f1329d23700 1 -- 172.21.15.159:0/197395940 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1324073ba0 msgr2=0x7f132410a610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:34.605 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.097+0000 7f1329d23700 1 --2- 172.21.15.159:0/197395940 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1324073ba0 0x7f132410a610 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7f130c0098a0 tx=0x7f130c009bb0 comp rx=0 tx=0).stop 2023-03-27T10:15:34.605 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.097+0000 7f1329d23700 1 -- 172.21.15.159:0/197395940 shutdown_connections 2023-03-27T10:15:34.605 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.097+0000 7f1329d23700 1 --2- 172.21.15.159:0/197395940 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1324073ba0 0x7f132410a610 unknown :-1 s=CLOSED pgs=3 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:34.605 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.097+0000 7f1329d23700 1 -- 172.21.15.159:0/197395940 >> 172.21.15.159:0/197395940 conn(0x7f13240ffba0 msgr2=0x7f1324101fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:34.606 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.097+0000 7f1329d23700 1 -- 172.21.15.159:0/197395940 shutdown_connections 2023-03-27T10:15:34.606 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.097+0000 7f1329d23700 1 -- 172.21.15.159:0/197395940 wait complete. 2023-03-27T10:15:34.606 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.097+0000 7f1329d23700 1 Processor -- start 2023-03-27T10:15:34.606 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.098+0000 7f1329d23700 1 -- start start 2023-03-27T10:15:34.606 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.098+0000 7f1329d23700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1324073ba0 0x7f132419ad30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:34.606 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.098+0000 7f1329d23700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f132419b270 con 0x7f1324073ba0 2023-03-27T10:15:34.606 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.098+0000 7f13237fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1324073ba0 0x7f132419ad30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:34.607 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.098+0000 7f13237fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1324073ba0 0x7f132419ad30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44328/0 (socket says 172.21.15.159:44328) 2023-03-27T10:15:34.607 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.098+0000 7f13237fe700 1 -- 172.21.15.159:0/189163519 learned_addr learned my addr 172.21.15.159:0/189163519 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:34.607 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.098+0000 7f13237fe700 1 -- 172.21.15.159:0/189163519 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f130c009550 con 0x7f1324073ba0 2023-03-27T10:15:34.607 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.098+0000 7f13237fe700 1 --2- 172.21.15.159:0/189163519 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1324073ba0 0x7f132419ad30 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7f130c004000 tx=0x7f130c004750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:34.607 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.098+0000 7f1320ff9700 1 -- 172.21.15.159:0/189163519 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f130c011630 con 0x7f1324073ba0 2023-03-27T10:15:34.607 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.098+0000 7f1320ff9700 1 -- 172.21.15.159:0/189163519 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f130c011790 con 0x7f1324073ba0 2023-03-27T10:15:34.607 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.098+0000 7f1320ff9700 1 -- 172.21.15.159:0/189163519 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f130c011a30 con 0x7f1324073ba0 2023-03-27T10:15:34.607 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.098+0000 7f1329d23700 1 -- 172.21.15.159:0/189163519 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f132410af90 con 0x7f1324073ba0 2023-03-27T10:15:34.608 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.099+0000 7f1329d23700 1 -- 172.21.15.159:0/189163519 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f132410ae00 con 0x7f1324073ba0 2023-03-27T10:15:34.608 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.099+0000 7f1320ff9700 1 -- 172.21.15.159:0/189163519 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 1) v1 ==== 656+0+0 (secure 0 0 0) 0x7f130c017020 con 0x7f1324073ba0 2023-03-27T10:15:34.608 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.099+0000 7f1320ff9700 1 -- 172.21.15.159:0/189163519 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f130c01c450 con 0x7f1324073ba0 2023-03-27T10:15:34.608 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.099+0000 7f1329d23700 1 -- 172.21.15.159:0/189163519 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f132410b120 con 0x7f1324073ba0 2023-03-27T10:15:34.608 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.102+0000 7f1320ff9700 1 -- 172.21.15.159:0/189163519 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72400 (secure 0 0 0) 0x7f130c01b450 con 0x7f1324073ba0 2023-03-27T10:15:34.608 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.169+0000 7f1329d23700 1 -- 172.21.15.159:0/189163519 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "config assimilate-conf"} v 0) v1 -- 0x7f1324194b70 con 0x7f1324073ba0 2023-03-27T10:15:34.608 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.172+0000 7f1320ff9700 1 -- 172.21.15.159:0/189163519 <== mon.0 v2:172.21.15.159:3300/0 7 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f130c01b5f0 con 0x7f1324073ba0 2023-03-27T10:15:34.608 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.172+0000 7f1320ff9700 1 -- 172.21.15.159:0/189163519 <== mon.0 v2:172.21.15.159:3300/0 8 ==== mon_command_ack([{"prefix": "config assimilate-conf"}]=0 v2) v1 ==== 70+0+456 (secure 0 0 0) 0x7f130c024630 con 0x7f1324073ba0 2023-03-27T10:15:34.609 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.173+0000 7f1329d23700 1 -- 172.21.15.159:0/189163519 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1324073ba0 msgr2=0x7f132419ad30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:34.609 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.173+0000 7f1329d23700 1 --2- 172.21.15.159:0/189163519 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1324073ba0 0x7f132419ad30 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7f130c004000 tx=0x7f130c004750 comp rx=0 tx=0).stop 2023-03-27T10:15:34.609 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.173+0000 7f1329d23700 1 -- 172.21.15.159:0/189163519 shutdown_connections 2023-03-27T10:15:34.609 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.173+0000 7f1329d23700 1 --2- 172.21.15.159:0/189163519 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1324073ba0 0x7f132419ad30 unknown :-1 s=CLOSED pgs=4 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:34.609 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.173+0000 7f1329d23700 1 -- 172.21.15.159:0/189163519 >> 172.21.15.159:0/189163519 conn(0x7f13240ffba0 msgr2=0x7f1324101460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:34.609 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.173+0000 7f1329d23700 1 -- 172.21.15.159:0/189163519 shutdown_connections 2023-03-27T10:15:34.609 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:34.173+0000 7f1329d23700 1 -- 172.21.15.159:0/189163519 wait complete. 2023-03-27T10:15:34.610 INFO:teuthology.orchestra.run.smithi159.stdout:Generating new minimal ceph.conf... 2023-03-27T10:15:34.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:34 smithi159 ceph-mon[92542]: from='client.? 172.21.15.159:0/189163519' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-03-27T10:15:34.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:34 smithi159 ceph-mon[92542]: from='client.? 172.21.15.159:0/189163519' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-03-27T10:15:35.546 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.073+0000 7fad1758b700 1 Processor -- start 2023-03-27T10:15:35.547 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.073+0000 7fad1758b700 1 -- start start 2023-03-27T10:15:35.547 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.074+0000 7fad1758b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fad10106180 0x7fad10108560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:35.547 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.074+0000 7fad1758b700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fad10073c20 con 0x7fad10106180 2023-03-27T10:15:35.547 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.074+0000 7fad15327700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fad10106180 0x7fad10108560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:35.547 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.074+0000 7fad15327700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fad10106180 0x7fad10108560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44330/0 (socket says 172.21.15.159:44330) 2023-03-27T10:15:35.548 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.074+0000 7fad15327700 1 -- 172.21.15.159:0/427799278 learned_addr learned my addr 172.21.15.159:0/427799278 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:35.548 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.074+0000 7fad15327700 1 -- 172.21.15.159:0/427799278 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fad10073d60 con 0x7fad10106180 2023-03-27T10:15:35.548 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.075+0000 7fad15327700 1 --2- 172.21.15.159:0/427799278 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fad10106180 0x7fad10108560 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7fad000098a0 tx=0x7fad00009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3fe1bffb7c7ff606 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:35.548 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.075+0000 7fad07fff700 1 -- 172.21.15.159:0/427799278 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fad00004160 con 0x7fad10106180 2023-03-27T10:15:35.548 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.075+0000 7fad07fff700 1 -- 172.21.15.159:0/427799278 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fad000042c0 con 0x7fad10106180 2023-03-27T10:15:35.548 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.075+0000 7fad07fff700 1 -- 172.21.15.159:0/427799278 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fad00011a00 con 0x7fad10106180 2023-03-27T10:15:35.548 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.075+0000 7fad1758b700 1 -- 172.21.15.159:0/427799278 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fad10106180 msgr2=0x7fad10108560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:35.548 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.075+0000 7fad1758b700 1 --2- 172.21.15.159:0/427799278 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fad10106180 0x7fad10108560 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7fad000098a0 tx=0x7fad00009bb0 comp rx=0 tx=0).stop 2023-03-27T10:15:35.549 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.075+0000 7fad1758b700 1 -- 172.21.15.159:0/427799278 shutdown_connections 2023-03-27T10:15:35.549 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.075+0000 7fad1758b700 1 --2- 172.21.15.159:0/427799278 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fad10106180 0x7fad10108560 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:35.549 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.075+0000 7fad1758b700 1 -- 172.21.15.159:0/427799278 >> 172.21.15.159:0/427799278 conn(0x7fad100ffba0 msgr2=0x7fad10101fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:35.549 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.075+0000 7fad1758b700 1 -- 172.21.15.159:0/427799278 shutdown_connections 2023-03-27T10:15:35.549 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.075+0000 7fad1758b700 1 -- 172.21.15.159:0/427799278 wait complete. 2023-03-27T10:15:35.549 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.076+0000 7fad1758b700 1 Processor -- start 2023-03-27T10:15:35.549 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.076+0000 7fad1758b700 1 -- start start 2023-03-27T10:15:35.550 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.076+0000 7fad1758b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fad10106180 0x7fad10199350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:35.550 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.076+0000 7fad1758b700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fad10199890 con 0x7fad10106180 2023-03-27T10:15:35.550 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.076+0000 7fad15327700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fad10106180 0x7fad10199350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:35.550 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.076+0000 7fad15327700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fad10106180 0x7fad10199350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44332/0 (socket says 172.21.15.159:44332) 2023-03-27T10:15:35.550 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.076+0000 7fad15327700 1 -- 172.21.15.159:0/2092107238 learned_addr learned my addr 172.21.15.159:0/2092107238 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:35.550 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.077+0000 7fad15327700 1 -- 172.21.15.159:0/2092107238 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fad00009550 con 0x7fad10106180 2023-03-27T10:15:35.550 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.077+0000 7fad15327700 1 --2- 172.21.15.159:0/2092107238 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fad10106180 0x7fad10199350 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7fad00009130 tx=0x7fad00011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:35.551 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.077+0000 7fad067fc700 1 -- 172.21.15.159:0/2092107238 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fad00003ab0 con 0x7fad10106180 2023-03-27T10:15:35.551 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.077+0000 7fad067fc700 1 -- 172.21.15.159:0/2092107238 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fad00003c10 con 0x7fad10106180 2023-03-27T10:15:35.551 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.077+0000 7fad1758b700 1 -- 172.21.15.159:0/2092107238 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fad101098b0 con 0x7fad10106180 2023-03-27T10:15:35.551 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.077+0000 7fad067fc700 1 -- 172.21.15.159:0/2092107238 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fad000247d0 con 0x7fad10106180 2023-03-27T10:15:35.551 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.077+0000 7fad1758b700 1 -- 172.21.15.159:0/2092107238 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fad10109720 con 0x7fad10106180 2023-03-27T10:15:35.551 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.077+0000 7fad067fc700 1 -- 172.21.15.159:0/2092107238 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 1) v1 ==== 656+0+0 (secure 0 0 0) 0x7fad00017020 con 0x7fad10106180 2023-03-27T10:15:35.551 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.077+0000 7fad067fc700 1 -- 172.21.15.159:0/2092107238 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7fad0002da10 con 0x7fad10106180 2023-03-27T10:15:35.551 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.078+0000 7fad1758b700 1 -- 172.21.15.159:0/2092107238 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fad10109a40 con 0x7fad10106180 2023-03-27T10:15:35.552 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.080+0000 7fad067fc700 1 -- 172.21.15.159:0/2092107238 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72400 (secure 0 0 0) 0x7fad0002d3f0 con 0x7fad10106180 2023-03-27T10:15:35.552 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.146+0000 7fad1758b700 1 -- 172.21.15.159:0/2092107238 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "config generate-minimal-conf"} v 0) v1 -- 0x7fad10199e50 con 0x7fad10106180 2023-03-27T10:15:35.552 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.147+0000 7fad067fc700 1 -- 172.21.15.159:0/2092107238 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "config generate-minimal-conf"}]=0 v2) v1 ==== 76+0+177 (secure 0 0 0) 0x7fad0001abb0 con 0x7fad10106180 2023-03-27T10:15:35.552 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.147+0000 7fad1758b700 1 -- 172.21.15.159:0/2092107238 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fad10106180 msgr2=0x7fad10199350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:35.552 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.147+0000 7fad1758b700 1 --2- 172.21.15.159:0/2092107238 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fad10106180 0x7fad10199350 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7fad00009130 tx=0x7fad00011ad0 comp rx=0 tx=0).stop 2023-03-27T10:15:35.552 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.147+0000 7fad1758b700 1 -- 172.21.15.159:0/2092107238 shutdown_connections 2023-03-27T10:15:35.552 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.147+0000 7fad1758b700 1 --2- 172.21.15.159:0/2092107238 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fad10106180 0x7fad10199350 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:35.552 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.147+0000 7fad1758b700 1 -- 172.21.15.159:0/2092107238 >> 172.21.15.159:0/2092107238 conn(0x7fad100ffba0 msgr2=0x7fad10100990 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:35.553 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.147+0000 7fad1758b700 1 -- 172.21.15.159:0/2092107238 shutdown_connections 2023-03-27T10:15:35.553 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:35.147+0000 7fad1758b700 1 -- 172.21.15.159:0/2092107238 wait complete. 2023-03-27T10:15:35.553 INFO:teuthology.orchestra.run.smithi159.stdout:Restarting the monitor... 2023-03-27T10:15:35.863 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:35 smithi159 systemd[1]: Stopping Ceph mon.a for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:15:35.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:35 smithi159 ceph-mon[92542]: from='client.? 172.21.15.159:0/2092107238' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:15:36.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:35 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mon-a[92517]: 2023-03-27T10:15:35.863+0000 7fb86d3cb700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:15:36.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:35 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mon-a[92517]: 2023-03-27T10:15:35.863+0000 7fb86d3cb700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-03-27T10:15:37.717 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:37 smithi159 bash[92877]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mon-a 2023-03-27T10:15:38.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:37 smithi159 systemd[1]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mon.a.service: Succeeded. 2023-03-27T10:15:38.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:37 smithi159 systemd[1]: Stopped Ceph mon.a for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:15:38.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:37 smithi159 systemd[1]: Starting Ceph mon.a for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:15:39.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:39 smithi159 podman[92998]: 2023-03-27T10:15:40.511 INFO:teuthology.orchestra.run.smithi159.stdout:Setting mon public_network to 172.21.0.0/20 2023-03-27T10:15:40.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: set uid:gid to 167:167 (ceph:ceph) 2023-03-27T10:15:40.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: ceph version 18.0.0-2533-gb70fdf97 (b70fdf973c2ffa23c0d8aa6918519da05d922302) reef (dev), process ceph-mon, pid 7 2023-03-27T10:15:40.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: pidfile_write: ignore empty --pid-file 2023-03-27T10:15:40.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: load: jerasure load: lrc 2023-03-27T10:15:40.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: RocksDB version: 6.15.5 2023-03-27T10:15:40.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-03-27T10:15:40.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Compile date Mar 16 2023 2023-03-27T10:15:40.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: DB SUMMARY 2023-03-27T10:15:40.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: DB Session ID: NCZCDP5JO01DORS8CTLH 2023-03-27T10:15:40.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: CURRENT file: CURRENT 2023-03-27T10:15:40.617 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: IDENTITY file: IDENTITY 2023-03-27T10:15:40.617 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-03-27T10:15:40.617 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-03-27T10:15:40.617 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 99147 ; 2023-03-27T10:15:40.617 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.error_if_exists: 0 2023-03-27T10:15:40.617 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.create_if_missing: 0 2023-03-27T10:15:40.617 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.paranoid_checks: 1 2023-03-27T10:15:40.618 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-03-27T10:15:40.618 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.env: 0x5654e90cf540 2023-03-27T10:15:40.618 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.fs: Posix File System 2023-03-27T10:15:40.618 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.info_log: 0x5654eac4e3a0 2023-03-27T10:15:40.618 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_file_opening_threads: 16 2023-03-27T10:15:40.618 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.statistics: (nil) 2023-03-27T10:15:40.618 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.use_fsync: 0 2023-03-27T10:15:40.618 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_log_file_size: 0 2023-03-27T10:15:40.619 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-03-27T10:15:40.619 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.log_file_time_to_roll: 0 2023-03-27T10:15:40.619 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.keep_log_file_num: 1000 2023-03-27T10:15:40.619 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.recycle_log_file_num: 0 2023-03-27T10:15:40.619 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.allow_fallocate: 1 2023-03-27T10:15:40.619 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.allow_mmap_reads: 0 2023-03-27T10:15:40.619 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.allow_mmap_writes: 0 2023-03-27T10:15:40.620 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.use_direct_reads: 0 2023-03-27T10:15:40.620 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-03-27T10:15:40.620 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.create_missing_column_families: 0 2023-03-27T10:15:40.620 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.db_log_dir: 2023-03-27T10:15:40.620 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-03-27T10:15:40.620 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.table_cache_numshardbits: 6 2023-03-27T10:15:40.620 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.WAL_ttl_seconds: 0 2023-03-27T10:15:40.620 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.WAL_size_limit_MB: 0 2023-03-27T10:15:40.621 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-03-27T10:15:40.621 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-03-27T10:15:40.621 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.is_fd_close_on_exec: 1 2023-03-27T10:15:40.621 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.advise_random_on_open: 1 2023-03-27T10:15:40.621 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.db_write_buffer_size: 0 2023-03-27T10:15:40.621 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.write_buffer_manager: 0x5654eaed6f60 2023-03-27T10:15:40.621 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-03-27T10:15:40.622 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-03-27T10:15:40.622 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-03-27T10:15:40.622 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.use_adaptive_mutex: 0 2023-03-27T10:15:40.622 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.rate_limiter: (nil) 2023-03-27T10:15:40.622 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-03-27T10:15:40.622 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.wal_recovery_mode: 2 2023-03-27T10:15:40.622 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.enable_thread_tracking: 0 2023-03-27T10:15:40.623 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.enable_pipelined_write: 0 2023-03-27T10:15:40.623 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.unordered_write: 0 2023-03-27T10:15:40.623 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-03-27T10:15:40.623 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-03-27T10:15:40.623 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-03-27T10:15:40.623 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-03-27T10:15:40.623 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.row_cache: None 2023-03-27T10:15:40.623 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.wal_filter: None 2023-03-27T10:15:40.624 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-03-27T10:15:40.624 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.allow_ingest_behind: 0 2023-03-27T10:15:40.624 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.preserve_deletes: 0 2023-03-27T10:15:40.624 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.two_write_queues: 0 2023-03-27T10:15:40.624 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.manual_wal_flush: 0 2023-03-27T10:15:40.624 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.atomic_flush: 0 2023-03-27T10:15:40.624 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-03-27T10:15:40.624 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.persist_stats_to_disk: 0 2023-03-27T10:15:40.625 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.write_dbid_to_manifest: 0 2023-03-27T10:15:40.625 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.log_readahead_size: 0 2023-03-27T10:15:40.625 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-03-27T10:15:40.625 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.best_efforts_recovery: 0 2023-03-27T10:15:40.625 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-03-27T10:15:40.625 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-03-27T10:15:40.625 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.allow_data_in_errors: 0 2023-03-27T10:15:40.626 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.db_host_id: __hostname__ 2023-03-27T10:15:40.626 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_background_jobs: 2 2023-03-27T10:15:40.627 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_background_compactions: -1 2023-03-27T10:15:40.627 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_subcompactions: 1 2023-03-27T10:15:40.627 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-03-27T10:15:40.627 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-03-27T10:15:40.627 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.delayed_write_rate : 16777216 2023-03-27T10:15:40.627 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_total_wal_size: 0 2023-03-27T10:15:40.628 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-03-27T10:15:40.628 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.stats_dump_period_sec: 600 2023-03-27T10:15:40.628 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.stats_persist_period_sec: 600 2023-03-27T10:15:40.628 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-03-27T10:15:40.628 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_open_files: -1 2023-03-27T10:15:40.628 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.bytes_per_sync: 0 2023-03-27T10:15:40.628 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.wal_bytes_per_sync: 0 2023-03-27T10:15:40.628 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.strict_bytes_per_sync: 0 2023-03-27T10:15:40.629 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compaction_readahead_size: 0 2023-03-27T10:15:40.629 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_background_flushes: -1 2023-03-27T10:15:40.629 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Compression algorithms supported: 2023-03-27T10:15:40.629 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-03-27T10:15:40.629 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: kZSTD supported: 0 2023-03-27T10:15:40.629 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: kXpressCompression supported: 0 2023-03-27T10:15:40.629 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: kLZ4HCCompression supported: 1 2023-03-27T10:15:40.630 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: kZlibCompression supported: 1 2023-03-27T10:15:40.630 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: kSnappyCompression supported: 1 2023-03-27T10:15:40.630 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: kLZ4Compression supported: 1 2023-03-27T10:15:40.630 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: kBZip2Compression supported: 0 2023-03-27T10:15:40.630 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Fast CRC32 supported: Supported on x86 2023-03-27T10:15:40.630 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-03-27T10:15:40.630 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-03-27T10:15:40.630 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-03-27T10:15:40.631 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.merge_operator: 2023-03-27T10:15:40.631 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compaction_filter: None 2023-03-27T10:15:40.631 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compaction_filter_factory: None 2023-03-27T10:15:40.631 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.sst_partitioner_factory: None 2023-03-27T10:15:40.631 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.memtable_factory: SkipListFactory 2023-03-27T10:15:40.631 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.table_factory: BlockBasedTable 2023-03-27T10:15:40.631 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5654eac5e5f0) 2023-03-27T10:15:40.631 INFO:journalctl@ceph.mon.a.smithi159.stdout: cache_index_and_filter_blocks: 1 2023-03-27T10:15:40.632 INFO:journalctl@ceph.mon.a.smithi159.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-03-27T10:15:40.632 INFO:journalctl@ceph.mon.a.smithi159.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-03-27T10:15:40.632 INFO:journalctl@ceph.mon.a.smithi159.stdout: pin_top_level_index_and_filter: 1 2023-03-27T10:15:40.632 INFO:journalctl@ceph.mon.a.smithi159.stdout: index_type: 0 2023-03-27T10:15:40.632 INFO:journalctl@ceph.mon.a.smithi159.stdout: data_block_index_type: 0 2023-03-27T10:15:40.632 INFO:journalctl@ceph.mon.a.smithi159.stdout: index_shortening: 1 2023-03-27T10:15:40.632 INFO:journalctl@ceph.mon.a.smithi159.stdout: data_block_hash_table_util_ratio: 0.750000 2023-03-27T10:15:40.632 INFO:journalctl@ceph.mon.a.smithi159.stdout: hash_index_allow_collision: 1 2023-03-27T10:15:40.633 INFO:journalctl@ceph.mon.a.smithi159.stdout: checksum: 1 2023-03-27T10:15:40.633 INFO:journalctl@ceph.mon.a.smithi159.stdout: no_block_cache: 0 2023-03-27T10:15:40.633 INFO:journalctl@ceph.mon.a.smithi159.stdout: block_cache: 0x5654eac9f350 2023-03-27T10:15:40.633 INFO:journalctl@ceph.mon.a.smithi159.stdout: block_cache_name: BinnedLRUCache 2023-03-27T10:15:40.633 INFO:journalctl@ceph.mon.a.smithi159.stdout: block_cache_options: 2023-03-27T10:15:40.633 INFO:journalctl@ceph.mon.a.smithi159.stdout: capacity : 536870912 2023-03-27T10:15:40.633 INFO:journalctl@ceph.mon.a.smithi159.stdout: num_shard_bits : 4 2023-03-27T10:15:40.633 INFO:journalctl@ceph.mon.a.smithi159.stdout: strict_capacity_limit : 0 2023-03-27T10:15:40.634 INFO:journalctl@ceph.mon.a.smithi159.stdout: high_pri_pool_ratio: 0.000 2023-03-27T10:15:40.634 INFO:journalctl@ceph.mon.a.smithi159.stdout: block_cache_compressed: (nil) 2023-03-27T10:15:40.634 INFO:journalctl@ceph.mon.a.smithi159.stdout: persistent_cache: (nil) 2023-03-27T10:15:40.634 INFO:journalctl@ceph.mon.a.smithi159.stdout: block_size: 4096 2023-03-27T10:15:40.634 INFO:journalctl@ceph.mon.a.smithi159.stdout: block_size_deviation: 10 2023-03-27T10:15:40.634 INFO:journalctl@ceph.mon.a.smithi159.stdout: block_restart_interval: 16 2023-03-27T10:15:40.634 INFO:journalctl@ceph.mon.a.smithi159.stdout: index_block_restart_interval: 1 2023-03-27T10:15:40.635 INFO:journalctl@ceph.mon.a.smithi159.stdout: metadata_block_size: 4096 2023-03-27T10:15:40.635 INFO:journalctl@ceph.mon.a.smithi159.stdout: partition_filters: 0 2023-03-27T10:15:40.635 INFO:journalctl@ceph.mon.a.smithi159.stdout: use_delta_encoding: 1 2023-03-27T10:15:40.635 INFO:journalctl@ceph.mon.a.smithi159.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-03-27T10:15:40.635 INFO:journalctl@ceph.mon.a.smithi159.stdout: whole_key_filtering: 1 2023-03-27T10:15:40.635 INFO:journalctl@ceph.mon.a.smithi159.stdout: verify_compression: 0 2023-03-27T10:15:40.635 INFO:journalctl@ceph.mon.a.smithi159.stdout: read_amp_bytes_per_bit: 0 2023-03-27T10:15:40.635 INFO:journalctl@ceph.mon.a.smithi159.stdout: format_version: 4 2023-03-27T10:15:40.636 INFO:journalctl@ceph.mon.a.smithi159.stdout: enable_index_compression: 1 2023-03-27T10:15:40.636 INFO:journalctl@ceph.mon.a.smithi159.stdout: block_align: 0 2023-03-27T10:15:40.636 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.write_buffer_size: 33554432 2023-03-27T10:15:40.636 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_write_buffer_number: 2 2023-03-27T10:15:40.636 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compression: NoCompression 2023-03-27T10:15:40.636 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.bottommost_compression: Disabled 2023-03-27T10:15:40.636 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.prefix_extractor: nullptr 2023-03-27T10:15:40.636 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-03-27T10:15:40.637 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.num_levels: 7 2023-03-27T10:15:40.637 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-03-27T10:15:40.637 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-03-27T10:15:40.637 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-03-27T10:15:40.637 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-03-27T10:15:40.637 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-03-27T10:15:40.637 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-03-27T10:15:40.637 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-03-27T10:15:40.638 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-03-27T10:15:40.638 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-03-27T10:15:40.638 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-03-27T10:15:40.638 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compression_opts.window_bits: -14 2023-03-27T10:15:40.638 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compression_opts.level: 32767 2023-03-27T10:15:40.638 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compression_opts.strategy: 0 2023-03-27T10:15:40.638 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-03-27T10:15:40.639 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-03-27T10:15:40.639 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-03-27T10:15:40.639 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compression_opts.enabled: false 2023-03-27T10:15:40.639 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-03-27T10:15:40.639 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-03-27T10:15:40.639 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-03-27T10:15:40.639 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.target_file_size_base: 67108864 2023-03-27T10:15:40.639 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.target_file_size_multiplier: 1 2023-03-27T10:15:40.640 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-03-27T10:15:40.640 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-03-27T10:15:40.640 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-03-27T10:15:40.640 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-03-27T10:15:40.640 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-03-27T10:15:40.640 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-03-27T10:15:40.640 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-03-27T10:15:40.640 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-03-27T10:15:40.641 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-03-27T10:15:40.641 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-03-27T10:15:40.641 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-03-27T10:15:40.641 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-03-27T10:15:40.641 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.arena_block_size: 4194304 2023-03-27T10:15:40.641 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-03-27T10:15:40.641 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-03-27T10:15:40.641 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-03-27T10:15:40.642 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.disable_auto_compactions: 0 2023-03-27T10:15:40.642 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-03-27T10:15:40.642 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-03-27T10:15:40.642 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-03-27T10:15:40.642 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-03-27T10:15:40.642 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-03-27T10:15:40.642 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-03-27T10:15:40.642 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-03-27T10:15:40.643 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-03-27T10:15:40.643 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-03-27T10:15:40.643 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-03-27T10:15:40.643 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.table_properties_collectors: 2023-03-27T10:15:40.643 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.inplace_update_support: 0 2023-03-27T10:15:40.643 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.inplace_update_num_locks: 10000 2023-03-27T10:15:40.643 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-03-27T10:15:40.643 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-03-27T10:15:40.644 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.memtable_huge_page_size: 0 2023-03-27T10:15:40.644 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.bloom_locality: 0 2023-03-27T10:15:40.644 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.max_successive_merges: 0 2023-03-27T10:15:40.644 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.optimize_filters_for_hits: 0 2023-03-27T10:15:40.644 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.paranoid_file_checks: 0 2023-03-27T10:15:40.644 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.force_consistency_checks: 1 2023-03-27T10:15:40.644 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.report_bg_io_stats: 0 2023-03-27T10:15:40.644 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.ttl: 2592000 2023-03-27T10:15:40.645 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.periodic_compaction_seconds: 0 2023-03-27T10:15:40.645 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.enable_blob_files: false 2023-03-27T10:15:40.645 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.min_blob_size: 0 2023-03-27T10:15:40.645 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.blob_file_size: 268435456 2023-03-27T10:15:40.645 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.blob_compression_type: NoCompression 2023-03-27T10:15:40.645 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.enable_blob_garbage_collection: false 2023-03-27T10:15:40.645 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-03-27T10:15:40.645 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-03-27T10:15:40.646 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-03-27T10:15:40.646 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-03-27T10:15:40.646 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: EVENT_LOG_v1 {"time_micros": 1679912140340690, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-03-27T10:15:40.646 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-03-27T10:15:40.646 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-03-27T10:15:40.646 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: EVENT_LOG_v1 {"time_micros": 1679912140341474, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 94271, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 92347, "index_size": 284, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 709, "raw_key_size": 13775, "raw_average_key_size": 50, "raw_value_size": 84833, "raw_average_value_size": 308, "num_data_blocks": 11, "num_entries": 275, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1679912140, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "8bcfba92-64e4-4c8d-9f0e-6a0e2ccfc78f", "db_session_id": "NCZCDP5JO01DORS8CTLH"}} 2023-03-27T10:15:40.646 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-03-27T10:15:40.647 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: EVENT_LOG_v1 {"time_micros": 1679912140341845, "job": 1, "event": "recovery_finished"} 2023-03-27T10:15:40.647 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-03-27T10:15:40.647 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5654eac33c00 2023-03-27T10:15:40.647 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: DB pointer 0x5654eac50000 2023-03-27T10:15:40.647 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-03-27T10:15:40.647 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-03-27T10:15:40.647 INFO:journalctl@ceph.mon.a.smithi159.stdout: ** DB Stats ** 2023-03-27T10:15:40.647 INFO:journalctl@ceph.mon.a.smithi159.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-03-27T10:15:40.648 INFO:journalctl@ceph.mon.a.smithi159.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-03-27T10:15:40.648 INFO:journalctl@ceph.mon.a.smithi159.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-03-27T10:15:40.648 INFO:journalctl@ceph.mon.a.smithi159.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-03-27T10:15:40.648 INFO:journalctl@ceph.mon.a.smithi159.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-03-27T10:15:40.648 INFO:journalctl@ceph.mon.a.smithi159.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-03-27T10:15:40.648 INFO:journalctl@ceph.mon.a.smithi159.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-03-27T10:15:40.648 INFO:journalctl@ceph.mon.a.smithi159.stdout: 2023-03-27T10:15:40.648 INFO:journalctl@ceph.mon.a.smithi159.stdout: ** Compaction Stats [default] ** 2023-03-27T10:15:40.649 INFO:journalctl@ceph.mon.a.smithi159.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-03-27T10:15:40.649 INFO:journalctl@ceph.mon.a.smithi159.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-03-27T10:15:40.649 INFO:journalctl@ceph.mon.a.smithi159.stdout: L0 2/0 93.82 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 174.2 0.00 0.00 1 0.001 0 0 2023-03-27T10:15:40.649 INFO:journalctl@ceph.mon.a.smithi159.stdout: Sum 2/0 93.82 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 174.2 0.00 0.00 1 0.001 0 0 2023-03-27T10:15:40.649 INFO:journalctl@ceph.mon.a.smithi159.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 174.2 0.00 0.00 1 0.001 0 0 2023-03-27T10:15:40.649 INFO:journalctl@ceph.mon.a.smithi159.stdout: 2023-03-27T10:15:40.649 INFO:journalctl@ceph.mon.a.smithi159.stdout: ** Compaction Stats [default] ** 2023-03-27T10:15:40.649 INFO:journalctl@ceph.mon.a.smithi159.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-03-27T10:15:40.650 INFO:journalctl@ceph.mon.a.smithi159.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-03-27T10:15:40.650 INFO:journalctl@ceph.mon.a.smithi159.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 174.2 0.00 0.00 1 0.001 0 0 2023-03-27T10:15:40.650 INFO:journalctl@ceph.mon.a.smithi159.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-03-27T10:15:40.650 INFO:journalctl@ceph.mon.a.smithi159.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-03-27T10:15:40.650 INFO:journalctl@ceph.mon.a.smithi159.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-03-27T10:15:40.650 INFO:journalctl@ceph.mon.a.smithi159.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-03-27T10:15:40.650 INFO:journalctl@ceph.mon.a.smithi159.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-03-27T10:15:40.651 INFO:journalctl@ceph.mon.a.smithi159.stdout: AddFile(Keys): cumulative 0, interval 0 2023-03-27T10:15:40.651 INFO:journalctl@ceph.mon.a.smithi159.stdout: Cumulative compaction: 0.00 GB write, 25.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-03-27T10:15:40.651 INFO:journalctl@ceph.mon.a.smithi159.stdout: Interval compaction: 0.00 GB write, 25.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-03-27T10:15:40.651 INFO:journalctl@ceph.mon.a.smithi159.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-03-27T10:15:40.651 INFO:journalctl@ceph.mon.a.smithi159.stdout: 2023-03-27T10:15:40.651 INFO:journalctl@ceph.mon.a.smithi159.stdout: ** File Read Latency Histogram By Level [default] ** 2023-03-27T10:15:40.651 INFO:journalctl@ceph.mon.a.smithi159.stdout: 2023-03-27T10:15:40.651 INFO:journalctl@ceph.mon.a.smithi159.stdout: ** Compaction Stats [default] ** 2023-03-27T10:15:40.652 INFO:journalctl@ceph.mon.a.smithi159.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-03-27T10:15:40.652 INFO:journalctl@ceph.mon.a.smithi159.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-03-27T10:15:40.652 INFO:journalctl@ceph.mon.a.smithi159.stdout: L0 2/0 93.82 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 174.2 0.00 0.00 1 0.001 0 0 2023-03-27T10:15:40.652 INFO:journalctl@ceph.mon.a.smithi159.stdout: Sum 2/0 93.82 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 174.2 0.00 0.00 1 0.001 0 0 2023-03-27T10:15:40.652 INFO:journalctl@ceph.mon.a.smithi159.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-03-27T10:15:40.652 INFO:journalctl@ceph.mon.a.smithi159.stdout: 2023-03-27T10:15:40.652 INFO:journalctl@ceph.mon.a.smithi159.stdout: ** Compaction Stats [default] ** 2023-03-27T10:15:40.653 INFO:journalctl@ceph.mon.a.smithi159.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-03-27T10:15:40.653 INFO:journalctl@ceph.mon.a.smithi159.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-03-27T10:15:40.653 INFO:journalctl@ceph.mon.a.smithi159.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 174.2 0.00 0.00 1 0.001 0 0 2023-03-27T10:15:40.653 INFO:journalctl@ceph.mon.a.smithi159.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-03-27T10:15:40.653 INFO:journalctl@ceph.mon.a.smithi159.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-03-27T10:15:40.653 INFO:journalctl@ceph.mon.a.smithi159.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-03-27T10:15:40.653 INFO:journalctl@ceph.mon.a.smithi159.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-03-27T10:15:40.654 INFO:journalctl@ceph.mon.a.smithi159.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-03-27T10:15:40.654 INFO:journalctl@ceph.mon.a.smithi159.stdout: AddFile(Keys): cumulative 0, interval 0 2023-03-27T10:15:40.654 INFO:journalctl@ceph.mon.a.smithi159.stdout: Cumulative compaction: 0.00 GB write, 25.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-03-27T10:15:40.654 INFO:journalctl@ceph.mon.a.smithi159.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-03-27T10:15:40.654 INFO:journalctl@ceph.mon.a.smithi159.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-03-27T10:15:40.654 INFO:journalctl@ceph.mon.a.smithi159.stdout: 2023-03-27T10:15:40.654 INFO:journalctl@ceph.mon.a.smithi159.stdout: ** File Read Latency Histogram By Level [default] ** 2023-03-27T10:15:40.654 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: starting mon.a rank 0 at public addrs [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] at bind addrs [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:15:40.655 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: mon.a@-1(???) e1 preinit fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:15:40.655 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: mon.a@-1(???).mds e1 new map 2023-03-27T10:15:40.655 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: mon.a@-1(???).mds e1 print_map 2023-03-27T10:15:40.655 INFO:journalctl@ceph.mon.a.smithi159.stdout: e1 2023-03-27T10:15:40.655 INFO:journalctl@ceph.mon.a.smithi159.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-03-27T10:15:40.655 INFO:journalctl@ceph.mon.a.smithi159.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-03-27T10:15:40.655 INFO:journalctl@ceph.mon.a.smithi159.stdout: legacy client fscid: -1 2023-03-27T10:15:40.656 INFO:journalctl@ceph.mon.a.smithi159.stdout: 2023-03-27T10:15:40.656 INFO:journalctl@ceph.mon.a.smithi159.stdout: No filesystems configured 2023-03-27T10:15:40.656 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-03-27T10:15:40.656 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-03-27T10:15:40.656 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-03-27T10:15:40.656 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-03-27T10:15:40.656 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-03-27T10:15:40.656 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: expand_channel_meta expand map: {default=false} 2023-03-27T10:15:40.657 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: expand_channel_meta from 'false' to 'false' 2023-03-27T10:15:40.657 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: expand_channel_meta expanded map: {default=false} 2023-03-27T10:15:40.657 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: expand_channel_meta expand map: {default=info} 2023-03-27T10:15:40.657 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: expand_channel_meta from 'info' to 'info' 2023-03-27T10:15:40.657 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: expand_channel_meta expanded map: {default=info} 2023-03-27T10:15:40.657 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: expand_channel_meta expand map: {default=daemon} 2023-03-27T10:15:40.657 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: expand_channel_meta from 'daemon' to 'daemon' 2023-03-27T10:15:40.657 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: expand_channel_meta expanded map: {default=daemon} 2023-03-27T10:15:40.658 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: expand_channel_meta expand map: {default=debug} 2023-03-27T10:15:40.658 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: expand_channel_meta from 'debug' to 'debug' 2023-03-27T10:15:40.658 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: expand_channel_meta expanded map: {default=debug} 2023-03-27T10:15:40.658 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: mon.a is new leader, mons a in quorum (ranks 0) 2023-03-27T10:15:40.658 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: monmap e1: 1 mons at {a=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0]} removed_ranks: {} 2023-03-27T10:15:40.658 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: fsmap 2023-03-27T10:15:40.658 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: osdmap e1: 0 total, 0 up, 0 in 2023-03-27T10:15:40.658 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 ceph-mon[93039]: mgrmap e1: no daemons active 2023-03-27T10:15:40.659 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 bash[92998]: 0b9f9cead7c8575fd83cc9b056b57e4d420124aeebce26e8c2a893db0f7acbf0 2023-03-27T10:15:40.659 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:40 smithi159 systemd[1]: Started Ceph mon.a for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:15:43.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:43 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/4139374787' entity='client.admin' 2023-03-27T10:15:44.873 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.442+0000 7f42d9054700 1 Processor -- start 2023-03-27T10:15:44.877 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.442+0000 7f42d9054700 1 -- start start 2023-03-27T10:15:44.877 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.443+0000 7f42d9054700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f42d4106d10 0x7f42d4107120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:44.877 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.443+0000 7f42d9054700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42d4107660 con 0x7f42d4106d10 2023-03-27T10:15:44.877 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.443+0000 7f42d2d9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f42d4106d10 0x7f42d4107120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:44.877 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.443+0000 7f42d2d9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f42d4106d10 0x7f42d4107120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44334/0 (socket says 172.21.15.159:44334) 2023-03-27T10:15:44.877 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.443+0000 7f42d2d9d700 1 -- 172.21.15.159:0/4016891119 learned_addr learned my addr 172.21.15.159:0/4016891119 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:44.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.443+0000 7f42d2d9d700 1 -- 172.21.15.159:0/4016891119 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f42d41077a0 con 0x7f42d4106d10 2023-03-27T10:15:44.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.443+0000 7f42d2d9d700 1 --2- 172.21.15.159:0/4016891119 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f42d4106d10 0x7f42d4107120 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f42bc0098a0 tx=0x7f42bc009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9a212fb6890497e5 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:44.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.444+0000 7f42d1d9b700 1 -- 172.21.15.159:0/4016891119 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f42bc004160 con 0x7f42d4106d10 2023-03-27T10:15:44.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.444+0000 7f42d1d9b700 1 -- 172.21.15.159:0/4016891119 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f42bc0042c0 con 0x7f42d4106d10 2023-03-27T10:15:44.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.444+0000 7f42d1d9b700 1 -- 172.21.15.159:0/4016891119 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f42bc011a00 con 0x7f42d4106d10 2023-03-27T10:15:44.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.444+0000 7f42d9054700 1 -- 172.21.15.159:0/4016891119 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f42d4106d10 msgr2=0x7f42d4107120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:44.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.444+0000 7f42d9054700 1 --2- 172.21.15.159:0/4016891119 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f42d4106d10 0x7f42d4107120 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f42bc0098a0 tx=0x7f42bc009bb0 comp rx=0 tx=0).stop 2023-03-27T10:15:44.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.444+0000 7f42d9054700 1 -- 172.21.15.159:0/4016891119 shutdown_connections 2023-03-27T10:15:44.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.444+0000 7f42d9054700 1 --2- 172.21.15.159:0/4016891119 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f42d4106d10 0x7f42d4107120 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:44.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.444+0000 7f42d9054700 1 -- 172.21.15.159:0/4016891119 >> 172.21.15.159:0/4016891119 conn(0x7f42d40ff3c0 msgr2=0x7f42d41017d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:44.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.444+0000 7f42d9054700 1 -- 172.21.15.159:0/4016891119 shutdown_connections 2023-03-27T10:15:44.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.444+0000 7f42d9054700 1 -- 172.21.15.159:0/4016891119 wait complete. 2023-03-27T10:15:44.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.445+0000 7f42d9054700 1 Processor -- start 2023-03-27T10:15:44.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.445+0000 7f42d9054700 1 -- start start 2023-03-27T10:15:44.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.445+0000 7f42d9054700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f42d4106d10 0x7f42d41992b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:44.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.445+0000 7f42d9054700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42d41997f0 con 0x7f42d4106d10 2023-03-27T10:15:44.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.446+0000 7f42d2d9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f42d4106d10 0x7f42d41992b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:44.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.446+0000 7f42d2d9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f42d4106d10 0x7f42d41992b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44336/0 (socket says 172.21.15.159:44336) 2023-03-27T10:15:44.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.446+0000 7f42d2d9d700 1 -- 172.21.15.159:0/4139374787 learned_addr learned my addr 172.21.15.159:0/4139374787 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:44.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.446+0000 7f42d2d9d700 1 -- 172.21.15.159:0/4139374787 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f42bc009550 con 0x7f42d4106d10 2023-03-27T10:15:44.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.446+0000 7f42d2d9d700 1 --2- 172.21.15.159:0/4139374787 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f42d4106d10 0x7f42d41992b0 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f42bc009fb0 tx=0x7f42bc011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:44.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.446+0000 7f42cbfff700 1 -- 172.21.15.159:0/4139374787 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f42bc003930 con 0x7f42d4106d10 2023-03-27T10:15:44.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.446+0000 7f42cbfff700 1 -- 172.21.15.159:0/4139374787 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f42bc003a90 con 0x7f42d4106d10 2023-03-27T10:15:44.881 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.446+0000 7f42cbfff700 1 -- 172.21.15.159:0/4139374787 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f42bc019610 con 0x7f42d4106d10 2023-03-27T10:15:44.881 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.446+0000 7f42d9054700 1 -- 172.21.15.159:0/4139374787 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f42d4104400 con 0x7f42d4106d10 2023-03-27T10:15:44.881 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.446+0000 7f42d9054700 1 -- 172.21.15.159:0/4139374787 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f42d4104270 con 0x7f42d4106d10 2023-03-27T10:15:44.881 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.446+0000 7f42cbfff700 1 -- 172.21.15.159:0/4139374787 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 1) v1 ==== 656+0+0 (secure 0 0 0) 0x7f42bc02a020 con 0x7f42d4106d10 2023-03-27T10:15:44.881 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.446+0000 7f42cbfff700 1 -- 172.21.15.159:0/4139374787 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f42bc02dc40 con 0x7f42d4106d10 2023-03-27T10:15:44.881 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.447+0000 7f42d9054700 1 -- 172.21.15.159:0/4139374787 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f42d4104590 con 0x7f42d4106d10 2023-03-27T10:15:44.881 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.450+0000 7f42cbfff700 1 -- 172.21.15.159:0/4139374787 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72400 (secure 0 0 0) 0x7f42bc003c00 con 0x7f42d4106d10 2023-03-27T10:15:44.881 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.518+0000 7f42d9054700 1 -- 172.21.15.159:0/4139374787 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command([{prefix=config set, name=public_network}] v 0) v1 -- 0x7f42d4061e00 con 0x7f42d4106d10 2023-03-27T10:15:44.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.521+0000 7f42cbfff700 1 -- 172.21.15.159:0/4139374787 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{prefix=config set, name=public_network}]=0 v3)=0 v3) v1 ==== 124+0+0 (secure 0 0 0) 0x7f42bc019d50 con 0x7f42d4106d10 2023-03-27T10:15:44.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.522+0000 7f42d9054700 1 -- 172.21.15.159:0/4139374787 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f42d4106d10 msgr2=0x7f42d41992b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:44.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.522+0000 7f42d9054700 1 --2- 172.21.15.159:0/4139374787 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f42d4106d10 0x7f42d41992b0 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f42bc009fb0 tx=0x7f42bc011ad0 comp rx=0 tx=0).stop 2023-03-27T10:15:44.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.522+0000 7f42d9054700 1 -- 172.21.15.159:0/4139374787 shutdown_connections 2023-03-27T10:15:44.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.522+0000 7f42d9054700 1 --2- 172.21.15.159:0/4139374787 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f42d4106d10 0x7f42d41992b0 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:44.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.522+0000 7f42d9054700 1 -- 172.21.15.159:0/4139374787 >> 172.21.15.159:0/4139374787 conn(0x7f42d40ff3c0 msgr2=0x7f42d4100090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:44.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.522+0000 7f42d9054700 1 -- 172.21.15.159:0/4139374787 shutdown_connections 2023-03-27T10:15:44.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:42.522+0000 7f42d9054700 1 -- 172.21.15.159:0/4139374787 wait complete. 2023-03-27T10:15:44.883 INFO:teuthology.orchestra.run.smithi159.stdout:Wrote config to /etc/ceph/ceph.conf 2023-03-27T10:15:44.883 INFO:teuthology.orchestra.run.smithi159.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-03-27T10:15:44.883 INFO:teuthology.orchestra.run.smithi159.stdout:Creating mgr... 2023-03-27T10:15:44.884 INFO:teuthology.orchestra.run.smithi159.stdout:Verifying port 9283 ... 2023-03-27T10:15:44.884 INFO:teuthology.orchestra.run.smithi159.stdout:Verifying port 8765 ... 2023-03-27T10:15:45.008 INFO:teuthology.orchestra.run.smithi159.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mgr.x 2023-03-27T10:15:45.008 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stderr Failed to reset failed state of unit ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mgr.x.service: Unit ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mgr.x.service not loaded. 2023-03-27T10:15:45.096 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c.target.wants/ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mgr.x.service → /etc/systemd/system/ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@.service. 2023-03-27T10:15:45.365 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:45 smithi159 systemd[1]: Starting Ceph mgr.x for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:15:45.688 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:45 smithi159 podman[93273]: 2023-03-27T10:15:45.688 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:45 smithi159 bash[93273]: 8e5ace82932d06aacfda859703fe41f88a4ba3bf03bb53969faff21e0a19d76b 2023-03-27T10:15:45.700 INFO:teuthology.orchestra.run.smithi159.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-03-27T10:15:45.700 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stdout disabled 2023-03-27T10:15:45.707 INFO:teuthology.orchestra.run.smithi159.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-03-27T10:15:45.707 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stdout inactive 2023-03-27T10:15:45.708 INFO:teuthology.orchestra.run.smithi159.stdout:firewalld.service is not enabled 2023-03-27T10:15:45.708 INFO:teuthology.orchestra.run.smithi159.stdout:Not possible to enable service . firewalld.service is not available 2023-03-27T10:15:45.717 INFO:teuthology.orchestra.run.smithi159.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-03-27T10:15:45.718 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stdout disabled 2023-03-27T10:15:45.726 INFO:teuthology.orchestra.run.smithi159.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-03-27T10:15:45.726 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stdout inactive 2023-03-27T10:15:45.726 INFO:teuthology.orchestra.run.smithi159.stdout:firewalld.service is not enabled 2023-03-27T10:15:45.726 INFO:teuthology.orchestra.run.smithi159.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2023-03-27T10:15:45.726 INFO:teuthology.orchestra.run.smithi159.stdout:Waiting for mgr to start... 2023-03-27T10:15:45.727 INFO:teuthology.orchestra.run.smithi159.stdout:Waiting for mgr... 2023-03-27T10:15:46.003 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:45 smithi159 systemd[1]: Started Ceph mgr.x for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:15:46.003 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:45 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:45.731+0000 7ff7d0df7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-03-27T10:15:46.003 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:45 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:45.818+0000 7ff7d0df7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-03-27T10:15:46.003 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:45 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:45.908+0000 7ff7d0df7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-03-27T10:15:46.364 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:46 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:46.004+0000 7ff7d0df7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-03-27T10:15:46.365 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:46 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:46.182+0000 7ff7d0df7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-03-27T10:15:46.731 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:46 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3510744346' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-03-27T10:15:46.732 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:46 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:46.513+0000 7ff7d0df7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-03-27T10:15:46.732 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:46 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:46.603+0000 7ff7d0df7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-03-27T10:15:46.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 2023-03-27T10:15:46.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout { 2023-03-27T10:15:46.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "fsid": "2c3cd140-cc88-11ed-9afe-001a4aab830c", 2023-03-27T10:15:46.883 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "health": { 2023-03-27T10:15:46.883 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-03-27T10:15:46.883 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-03-27T10:15:46.883 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-03-27T10:15:46.883 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:46.883 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-03-27T10:15:46.883 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-03-27T10:15:46.883 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 0 2023-03-27T10:15:46.884 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout ], 2023-03-27T10:15:46.884 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-03-27T10:15:46.884 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "a" 2023-03-27T10:15:46.884 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout ], 2023-03-27T10:15:46.884 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2023-03-27T10:15:46.884 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "monmap": { 2023-03-27T10:15:46.885 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-03-27T10:15:46.885 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2023-03-27T10:15:46.885 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-03-27T10:15:46.885 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:46.885 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-03-27T10:15:46.885 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-03-27T10:15:46.885 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-03-27T10:15:46.886 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-03-27T10:15:46.886 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-03-27T10:15:46.886 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-03-27T10:15:46.886 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-03-27T10:15:46.886 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-03-27T10:15:46.886 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:46.886 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-03-27T10:15:46.886 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-03-27T10:15:46.887 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-03-27T10:15:46.887 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-03-27T10:15:46.887 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-03-27T10:15:46.887 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-03-27T10:15:46.887 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-03-27T10:15:46.887 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-03-27T10:15:46.887 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-03-27T10:15:46.888 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:46.888 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-03-27T10:15:46.888 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-03-27T10:15:46.888 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-03-27T10:15:46.888 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-03-27T10:15:46.888 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:46.888 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-03-27T10:15:46.888 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "available": false, 2023-03-27T10:15:46.889 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-03-27T10:15:46.889 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "modules": [ 2023-03-27T10:15:46.889 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "iostat", 2023-03-27T10:15:46.889 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "nfs", 2023-03-27T10:15:46.889 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "restful" 2023-03-27T10:15:46.889 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout ], 2023-03-27T10:15:46.889 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "services": {} 2023-03-27T10:15:46.889 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:46.890 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-03-27T10:15:46.890 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-03-27T10:15:46.890 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "modified": "2023-03-27T10:15:32.617699+0000", 2023-03-27T10:15:46.890 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "services": {} 2023-03-27T10:15:46.890 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:46.890 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-03-27T10:15:46.890 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout } 2023-03-27T10:15:46.890 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.315+0000 7f5615604700 1 Processor -- start 2023-03-27T10:15:46.891 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.315+0000 7f5615604700 1 -- start start 2023-03-27T10:15:46.891 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.315+0000 7f5615604700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5610105f90 0x7f5610108370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:46.891 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.315+0000 7f5615604700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5610073c40 con 0x7f5610105f90 2023-03-27T10:15:46.891 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.315+0000 7f560effd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5610105f90 0x7f5610108370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:46.891 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.315+0000 7f560effd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5610105f90 0x7f5610108370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44342/0 (socket says 172.21.15.159:44342) 2023-03-27T10:15:46.891 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.315+0000 7f560effd700 1 -- 172.21.15.159:0/3937523730 learned_addr learned my addr 172.21.15.159:0/3937523730 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:46.891 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.316+0000 7f560effd700 1 -- 172.21.15.159:0/3937523730 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5610073d80 con 0x7f5610105f90 2023-03-27T10:15:46.891 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.316+0000 7f560effd700 1 --2- 172.21.15.159:0/3937523730 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5610105f90 0x7f5610108370 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f56000098a0 tx=0x7f5600009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dfd6214346ee9e1a server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:46.892 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.316+0000 7f560dffb700 1 -- 172.21.15.159:0/3937523730 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5600004160 con 0x7f5610105f90 2023-03-27T10:15:46.892 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.316+0000 7f560dffb700 1 -- 172.21.15.159:0/3937523730 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f56000042c0 con 0x7f5610105f90 2023-03-27T10:15:46.892 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.316+0000 7f560dffb700 1 -- 172.21.15.159:0/3937523730 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5600011a00 con 0x7f5610105f90 2023-03-27T10:15:46.892 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.317+0000 7f5615604700 1 -- 172.21.15.159:0/3937523730 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5610105f90 msgr2=0x7f5610108370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:46.892 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.317+0000 7f5615604700 1 --2- 172.21.15.159:0/3937523730 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5610105f90 0x7f5610108370 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f56000098a0 tx=0x7f5600009bb0 comp rx=0 tx=0).stop 2023-03-27T10:15:46.892 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.317+0000 7f5615604700 1 -- 172.21.15.159:0/3937523730 shutdown_connections 2023-03-27T10:15:46.892 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.317+0000 7f5615604700 1 --2- 172.21.15.159:0/3937523730 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5610105f90 0x7f5610108370 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:46.893 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.317+0000 7f5615604700 1 -- 172.21.15.159:0/3937523730 >> 172.21.15.159:0/3937523730 conn(0x7f56100ffa10 msgr2=0x7f5610101e20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:46.893 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.317+0000 7f5615604700 1 -- 172.21.15.159:0/3937523730 shutdown_connections 2023-03-27T10:15:46.893 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.317+0000 7f5615604700 1 -- 172.21.15.159:0/3937523730 wait complete. 2023-03-27T10:15:46.893 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.317+0000 7f5615604700 1 Processor -- start 2023-03-27T10:15:46.893 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.317+0000 7f5615604700 1 -- start start 2023-03-27T10:15:46.893 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.318+0000 7f5615604700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5610105f90 0x7f56101058f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:46.893 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.318+0000 7f5615604700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5610105e30 con 0x7f5610105f90 2023-03-27T10:15:46.893 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.318+0000 7f560effd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5610105f90 0x7f56101058f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:46.894 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.318+0000 7f560effd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5610105f90 0x7f56101058f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44344/0 (socket says 172.21.15.159:44344) 2023-03-27T10:15:46.894 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.318+0000 7f560effd700 1 -- 172.21.15.159:0/3510744346 learned_addr learned my addr 172.21.15.159:0/3510744346 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:46.894 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.318+0000 7f560effd700 1 -- 172.21.15.159:0/3510744346 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5600009550 con 0x7f5610105f90 2023-03-27T10:15:46.894 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.318+0000 7f560effd700 1 --2- 172.21.15.159:0/3510744346 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5610105f90 0x7f56101058f0 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f5600006af0 tx=0x7f5600011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:46.894 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.318+0000 7f55f7fff700 1 -- 172.21.15.159:0/3510744346 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5600003a40 con 0x7f5610105f90 2023-03-27T10:15:46.894 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.318+0000 7f55f7fff700 1 -- 172.21.15.159:0/3510744346 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f5600003ba0 con 0x7f5610105f90 2023-03-27T10:15:46.895 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.318+0000 7f5615604700 1 -- 172.21.15.159:0/3510744346 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5610109710 con 0x7f5610105f90 2023-03-27T10:15:46.895 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.318+0000 7f55f7fff700 1 -- 172.21.15.159:0/3510744346 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f560001a700 con 0x7f5610105f90 2023-03-27T10:15:46.895 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.318+0000 7f5615604700 1 -- 172.21.15.159:0/3510744346 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5610109580 con 0x7f5610105f90 2023-03-27T10:15:46.895 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.319+0000 7f55f7fff700 1 -- 172.21.15.159:0/3510744346 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 1) v1 ==== 656+0+0 (secure 0 0 0) 0x7f5600017020 con 0x7f5610105f90 2023-03-27T10:15:46.895 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.319+0000 7f55f7fff700 1 -- 172.21.15.159:0/3510744346 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f56000193c0 con 0x7f5610105f90 2023-03-27T10:15:46.895 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.320+0000 7f5615604700 1 -- 172.21.15.159:0/3510744346 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5610061e00 con 0x7f5610105f90 2023-03-27T10:15:46.895 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.322+0000 7f55f7fff700 1 -- 172.21.15.159:0/3510744346 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72400 (secure 0 0 0) 0x7f5600024ad0 con 0x7f5610105f90 2023-03-27T10:15:46.896 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.394+0000 7f5615604700 1 -- 172.21.15.159:0/3510744346 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7f5610104d80 con 0x7f5610105f90 2023-03-27T10:15:46.896 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.395+0000 7f55f7fff700 1 -- 172.21.15.159:0/3510744346 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1238 (secure 0 0 0) 0x7f5600024c70 con 0x7f5610105f90 2023-03-27T10:15:46.896 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.396+0000 7f5615604700 1 -- 172.21.15.159:0/3510744346 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5610105f90 msgr2=0x7f56101058f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:46.896 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.396+0000 7f5615604700 1 --2- 172.21.15.159:0/3510744346 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5610105f90 0x7f56101058f0 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f5600006af0 tx=0x7f5600011ad0 comp rx=0 tx=0).stop 2023-03-27T10:15:46.896 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.396+0000 7f5615604700 1 -- 172.21.15.159:0/3510744346 shutdown_connections 2023-03-27T10:15:46.896 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.396+0000 7f5615604700 1 --2- 172.21.15.159:0/3510744346 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5610105f90 0x7f56101058f0 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:46.896 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.396+0000 7f5615604700 1 -- 172.21.15.159:0/3510744346 >> 172.21.15.159:0/3510744346 conn(0x7f56100ffa10 msgr2=0x7f5610100770 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:46.896 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.396+0000 7f5615604700 1 -- 172.21.15.159:0/3510744346 shutdown_connections 2023-03-27T10:15:46.897 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:46.396+0000 7f5615604700 1 -- 172.21.15.159:0/3510744346 wait complete. 2023-03-27T10:15:46.897 INFO:teuthology.orchestra.run.smithi159.stdout:mgr not available, waiting (1/15)... 2023-03-27T10:15:47.115 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:46 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:46.732+0000 7ff7d0df7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-03-27T10:15:47.115 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:46 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:46.826+0000 7ff7d0df7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-03-27T10:15:47.614 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:47 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:47.256+0000 7ff7d0df7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-03-27T10:15:48.114 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:47 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:47.829+0000 7ff7d0df7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-03-27T10:15:48.115 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:47 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:47.997+0000 7ff7d0df7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-03-27T10:15:49.012 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:48 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:48.748+0000 7ff7d0df7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-03-27T10:15:49.012 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:48 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:48.837+0000 7ff7d0df7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-03-27T10:15:49.358 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:49 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:49.013+0000 7ff7d0df7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-03-27T10:15:49.358 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:49 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:49.205+0000 7ff7d0df7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-03-27T10:15:49.614 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:49 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:49.359+0000 7ff7d0df7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-03-27T10:15:49.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:49 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/1772964379' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-03-27T10:15:49.725 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 2023-03-27T10:15:49.727 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout { 2023-03-27T10:15:49.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "fsid": "2c3cd140-cc88-11ed-9afe-001a4aab830c", 2023-03-27T10:15:49.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "health": { 2023-03-27T10:15:49.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-03-27T10:15:49.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-03-27T10:15:49.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-03-27T10:15:49.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:49.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-03-27T10:15:49.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-03-27T10:15:49.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 0 2023-03-27T10:15:49.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout ], 2023-03-27T10:15:49.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-03-27T10:15:49.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "a" 2023-03-27T10:15:49.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout ], 2023-03-27T10:15:49.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2023-03-27T10:15:49.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "monmap": { 2023-03-27T10:15:49.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-03-27T10:15:49.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2023-03-27T10:15:49.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-03-27T10:15:49.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:49.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-03-27T10:15:49.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-03-27T10:15:49.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-03-27T10:15:49.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-03-27T10:15:49.731 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-03-27T10:15:49.736 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-03-27T10:15:49.736 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-03-27T10:15:49.737 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-03-27T10:15:49.737 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:49.737 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-03-27T10:15:49.737 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-03-27T10:15:49.737 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-03-27T10:15:49.737 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-03-27T10:15:49.737 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-03-27T10:15:49.738 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-03-27T10:15:49.738 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-03-27T10:15:49.738 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-03-27T10:15:49.738 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-03-27T10:15:49.738 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:49.738 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-03-27T10:15:49.738 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-03-27T10:15:49.738 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-03-27T10:15:49.739 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-03-27T10:15:49.739 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:49.739 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-03-27T10:15:49.739 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "available": false, 2023-03-27T10:15:49.739 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-03-27T10:15:49.739 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "modules": [ 2023-03-27T10:15:49.739 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "iostat", 2023-03-27T10:15:49.739 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "nfs", 2023-03-27T10:15:49.740 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "restful" 2023-03-27T10:15:49.740 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout ], 2023-03-27T10:15:49.740 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "services": {} 2023-03-27T10:15:49.740 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:49.740 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-03-27T10:15:49.740 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-03-27T10:15:49.740 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "modified": "2023-03-27T10:15:32.617699+0000", 2023-03-27T10:15:49.740 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "services": {} 2023-03-27T10:15:49.741 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:49.741 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-03-27T10:15:49.741 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout } 2023-03-27T10:15:49.741 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.339+0000 7f823be43700 1 Processor -- start 2023-03-27T10:15:49.741 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.340+0000 7f823be43700 1 -- start start 2023-03-27T10:15:49.741 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.340+0000 7f823be43700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82341056c0 0x7f8234105ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:49.741 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.340+0000 7f823be43700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8234106010 con 0x7f82341056c0 2023-03-27T10:15:49.742 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.340+0000 7f8239bdf700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82341056c0 0x7f8234105ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:49.742 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.340+0000 7f8239bdf700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82341056c0 0x7f8234105ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44346/0 (socket says 172.21.15.159:44346) 2023-03-27T10:15:49.742 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.340+0000 7f8239bdf700 1 -- 172.21.15.159:0/2639927844 learned_addr learned my addr 172.21.15.159:0/2639927844 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:49.742 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.340+0000 7f8239bdf700 1 -- 172.21.15.159:0/2639927844 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8234106150 con 0x7f82341056c0 2023-03-27T10:15:49.742 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.340+0000 7f8239bdf700 1 --2- 172.21.15.159:0/2639927844 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82341056c0 0x7f8234105ad0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f82300098a0 tx=0x7f8230009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=484b04a649119abc server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:49.742 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.341+0000 7f8238bdd700 1 -- 172.21.15.159:0/2639927844 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f8230004160 con 0x7f82341056c0 2023-03-27T10:15:49.742 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.341+0000 7f8238bdd700 1 -- 172.21.15.159:0/2639927844 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f82300042c0 con 0x7f82341056c0 2023-03-27T10:15:49.742 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.341+0000 7f8238bdd700 1 -- 172.21.15.159:0/2639927844 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f8230011a00 con 0x7f82341056c0 2023-03-27T10:15:49.743 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.341+0000 7f823be43700 1 -- 172.21.15.159:0/2639927844 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82341056c0 msgr2=0x7f8234105ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:49.743 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.341+0000 7f823be43700 1 --2- 172.21.15.159:0/2639927844 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82341056c0 0x7f8234105ad0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f82300098a0 tx=0x7f8230009bb0 comp rx=0 tx=0).stop 2023-03-27T10:15:49.743 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.341+0000 7f823be43700 1 -- 172.21.15.159:0/2639927844 shutdown_connections 2023-03-27T10:15:49.743 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.341+0000 7f823be43700 1 --2- 172.21.15.159:0/2639927844 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82341056c0 0x7f8234105ad0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:49.743 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.341+0000 7f823be43700 1 -- 172.21.15.159:0/2639927844 >> 172.21.15.159:0/2639927844 conn(0x7f8234100d10 msgr2=0x7f8234103140 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:49.743 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.341+0000 7f823be43700 1 -- 172.21.15.159:0/2639927844 shutdown_connections 2023-03-27T10:15:49.743 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.341+0000 7f823be43700 1 -- 172.21.15.159:0/2639927844 wait complete. 2023-03-27T10:15:49.744 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.342+0000 7f823be43700 1 Processor -- start 2023-03-27T10:15:49.744 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.342+0000 7f823be43700 1 -- start start 2023-03-27T10:15:49.744 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.342+0000 7f823be43700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82341056c0 0x7f823407aac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:49.744 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.342+0000 7f823be43700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f823407b000 con 0x7f82341056c0 2023-03-27T10:15:49.744 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.342+0000 7f8239bdf700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82341056c0 0x7f823407aac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:49.744 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.342+0000 7f8239bdf700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82341056c0 0x7f823407aac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44348/0 (socket says 172.21.15.159:44348) 2023-03-27T10:15:49.744 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.342+0000 7f8239bdf700 1 -- 172.21.15.159:0/1772964379 learned_addr learned my addr 172.21.15.159:0/1772964379 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:49.744 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.343+0000 7f8239bdf700 1 -- 172.21.15.159:0/1772964379 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8230009550 con 0x7f82341056c0 2023-03-27T10:15:49.745 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.343+0000 7f8239bdf700 1 --2- 172.21.15.159:0/1772964379 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82341056c0 0x7f823407aac0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f8230006af0 tx=0x7f8230011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:49.745 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.343+0000 7f822affd700 1 -- 172.21.15.159:0/1772964379 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f8230003a40 con 0x7f82341056c0 2023-03-27T10:15:49.745 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.343+0000 7f822affd700 1 -- 172.21.15.159:0/1772964379 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f8230003ba0 con 0x7f82341056c0 2023-03-27T10:15:49.745 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.343+0000 7f822affd700 1 -- 172.21.15.159:0/1772964379 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f823001a6e0 con 0x7f82341056c0 2023-03-27T10:15:49.745 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.343+0000 7f823be43700 1 -- 172.21.15.159:0/1772964379 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8234107710 con 0x7f82341056c0 2023-03-27T10:15:49.745 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.343+0000 7f823be43700 1 -- 172.21.15.159:0/1772964379 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8234107580 con 0x7f82341056c0 2023-03-27T10:15:49.745 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.343+0000 7f822affd700 1 -- 172.21.15.159:0/1772964379 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 1) v1 ==== 656+0+0 (secure 0 0 0) 0x7f8230017020 con 0x7f82341056c0 2023-03-27T10:15:49.746 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.343+0000 7f822affd700 1 -- 172.21.15.159:0/1772964379 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f82300193c0 con 0x7f82341056c0 2023-03-27T10:15:49.746 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.344+0000 7f8228ff9700 1 -- 172.21.15.159:0/1772964379 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8234061e00 con 0x7f82341056c0 2023-03-27T10:15:49.746 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.346+0000 7f822affd700 1 -- 172.21.15.159:0/1772964379 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72400 (secure 0 0 0) 0x7f8230024a60 con 0x7f82341056c0 2023-03-27T10:15:49.746 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.419+0000 7f8228ff9700 1 -- 172.21.15.159:0/1772964379 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7f82341a0ac0 con 0x7f82341056c0 2023-03-27T10:15:49.746 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.419+0000 7f822affd700 1 -- 172.21.15.159:0/1772964379 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1238 (secure 0 0 0) 0x7f8230035090 con 0x7f82341056c0 2023-03-27T10:15:49.746 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.420+0000 7f8228ff9700 1 -- 172.21.15.159:0/1772964379 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82341056c0 msgr2=0x7f823407aac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:49.746 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.420+0000 7f8228ff9700 1 --2- 172.21.15.159:0/1772964379 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82341056c0 0x7f823407aac0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f8230006af0 tx=0x7f8230011ad0 comp rx=0 tx=0).stop 2023-03-27T10:15:49.747 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.420+0000 7f8228ff9700 1 -- 172.21.15.159:0/1772964379 shutdown_connections 2023-03-27T10:15:49.747 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.420+0000 7f8228ff9700 1 --2- 172.21.15.159:0/1772964379 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82341056c0 0x7f823407aac0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:49.747 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.420+0000 7f8228ff9700 1 -- 172.21.15.159:0/1772964379 >> 172.21.15.159:0/1772964379 conn(0x7f8234100d10 msgr2=0x7f82341019e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:49.747 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.420+0000 7f8228ff9700 1 -- 172.21.15.159:0/1772964379 shutdown_connections 2023-03-27T10:15:49.747 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:49.420+0000 7f8228ff9700 1 -- 172.21.15.159:0/1772964379 wait complete. 2023-03-27T10:15:49.747 INFO:teuthology.orchestra.run.smithi159.stdout:mgr not available, waiting (2/15)... 2023-03-27T10:15:50.114 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:49 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:49.769+0000 7ff7d0df7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-03-27T10:15:50.600 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:50 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:50.294+0000 7ff7d0df7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-03-27T10:15:50.864 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:50 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:50.602+0000 7ff7d0df7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-03-27T10:15:50.865 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:50 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:50.786+0000 7ff7d0df7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-03-27T10:15:51.364 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:50 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:50.898+0000 7ff7d0df7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-03-27T10:15:51.365 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:51 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:51.107+0000 7ff7d0df7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-03-27T10:15:52.304 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:51 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:51.962+0000 7ff7d0df7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-03-27T10:15:52.305 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:52 smithi159 ceph-mon[93039]: Activating manager daemon x 2023-03-27T10:15:52.305 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:52 smithi159 ceph-mon[93039]: mgrmap e2: x(active, starting, since 0.00308935s) 2023-03-27T10:15:52.305 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:52 smithi159 ceph-mon[93039]: from='mgr.14100 172.21.15.159:0/3660332349' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-03-27T10:15:52.305 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:52 smithi159 ceph-mon[93039]: from='mgr.14100 172.21.15.159:0/3660332349' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-03-27T10:15:52.306 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:52 smithi159 ceph-mon[93039]: from='mgr.14100 172.21.15.159:0/3660332349' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-03-27T10:15:52.306 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:52 smithi159 ceph-mon[93039]: from='mgr.14100 172.21.15.159:0/3660332349' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:15:52.306 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:52 smithi159 ceph-mon[93039]: from='mgr.14100 172.21.15.159:0/3660332349' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-03-27T10:15:52.306 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:52 smithi159 ceph-mon[93039]: Manager daemon x is now available 2023-03-27T10:15:52.306 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:52 smithi159 ceph-mon[93039]: from='mgr.14100 172.21.15.159:0/3660332349' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-03-27T10:15:52.306 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:52 smithi159 ceph-mon[93039]: from='mgr.14100 172.21.15.159:0/3660332349' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-03-27T10:15:52.306 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:52 smithi159 ceph-mon[93039]: from='mgr.14100 172.21.15.159:0/3660332349' entity='mgr.x' 2023-03-27T10:15:52.306 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:52 smithi159 ceph-mon[93039]: from='mgr.14100 172.21.15.159:0/3660332349' entity='mgr.x' 2023-03-27T10:15:52.677 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 2023-03-27T10:15:52.677 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout { 2023-03-27T10:15:52.677 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "fsid": "2c3cd140-cc88-11ed-9afe-001a4aab830c", 2023-03-27T10:15:52.677 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "health": { 2023-03-27T10:15:52.677 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-03-27T10:15:52.678 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-03-27T10:15:52.678 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-03-27T10:15:52.678 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:52.678 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-03-27T10:15:52.678 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-03-27T10:15:52.678 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 0 2023-03-27T10:15:52.678 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout ], 2023-03-27T10:15:52.679 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-03-27T10:15:52.679 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "a" 2023-03-27T10:15:52.679 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout ], 2023-03-27T10:15:52.679 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "quorum_age": 11, 2023-03-27T10:15:52.679 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "monmap": { 2023-03-27T10:15:52.679 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-03-27T10:15:52.680 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2023-03-27T10:15:52.680 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-03-27T10:15:52.680 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:52.680 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-03-27T10:15:52.680 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-03-27T10:15:52.680 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-03-27T10:15:52.681 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-03-27T10:15:52.681 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-03-27T10:15:52.686 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-03-27T10:15:52.686 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-03-27T10:15:52.687 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-03-27T10:15:52.687 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:52.687 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-03-27T10:15:52.687 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-03-27T10:15:52.687 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-03-27T10:15:52.687 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-03-27T10:15:52.687 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-03-27T10:15:52.687 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-03-27T10:15:52.688 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-03-27T10:15:52.688 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-03-27T10:15:52.688 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-03-27T10:15:52.688 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:52.688 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-03-27T10:15:52.688 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-03-27T10:15:52.689 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-03-27T10:15:52.689 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-03-27T10:15:52.689 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:52.689 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-03-27T10:15:52.689 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "available": false, 2023-03-27T10:15:52.689 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-03-27T10:15:52.689 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "modules": [ 2023-03-27T10:15:52.690 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "iostat", 2023-03-27T10:15:52.690 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "nfs", 2023-03-27T10:15:52.690 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "restful" 2023-03-27T10:15:52.690 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout ], 2023-03-27T10:15:52.690 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "services": {} 2023-03-27T10:15:52.690 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:52.690 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-03-27T10:15:52.690 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-03-27T10:15:52.691 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "modified": "2023-03-27T10:15:32.617699+0000", 2023-03-27T10:15:52.691 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "services": {} 2023-03-27T10:15:52.691 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:52.691 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-03-27T10:15:52.691 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout } 2023-03-27T10:15:52.691 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.204+0000 7f2b0273b700 1 Processor -- start 2023-03-27T10:15:52.691 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.204+0000 7f2b0273b700 1 -- start start 2023-03-27T10:15:52.692 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.204+0000 7f2b0273b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2afc1061a0 0x7f2afc108580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:52.692 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.204+0000 7f2b0273b700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2afc073c00 con 0x7f2afc1061a0 2023-03-27T10:15:52.692 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.204+0000 7f2afbfff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2afc1061a0 0x7f2afc108580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:52.692 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.204+0000 7f2afbfff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2afc1061a0 0x7f2afc108580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44364/0 (socket says 172.21.15.159:44364) 2023-03-27T10:15:52.692 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.204+0000 7f2afbfff700 1 -- 172.21.15.159:0/1153479070 learned_addr learned my addr 172.21.15.159:0/1153479070 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:52.692 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.205+0000 7f2afbfff700 1 -- 172.21.15.159:0/1153479070 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2afc073d40 con 0x7f2afc1061a0 2023-03-27T10:15:52.692 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.205+0000 7f2afbfff700 1 --2- 172.21.15.159:0/1153479070 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2afc1061a0 0x7f2afc108580 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f2aec0098a0 tx=0x7f2aec009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=37cd034989f24c48 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:52.692 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.205+0000 7f2afaffd700 1 -- 172.21.15.159:0/1153479070 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f2aec004160 con 0x7f2afc1061a0 2023-03-27T10:15:52.693 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.205+0000 7f2afaffd700 1 -- 172.21.15.159:0/1153479070 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f2aec0042c0 con 0x7f2afc1061a0 2023-03-27T10:15:52.693 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.205+0000 7f2afaffd700 1 -- 172.21.15.159:0/1153479070 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f2aec011a00 con 0x7f2afc1061a0 2023-03-27T10:15:52.693 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.205+0000 7f2b0273b700 1 -- 172.21.15.159:0/1153479070 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2afc1061a0 msgr2=0x7f2afc108580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:52.693 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.205+0000 7f2b0273b700 1 --2- 172.21.15.159:0/1153479070 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2afc1061a0 0x7f2afc108580 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f2aec0098a0 tx=0x7f2aec009bb0 comp rx=0 tx=0).stop 2023-03-27T10:15:52.693 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.206+0000 7f2b0273b700 1 -- 172.21.15.159:0/1153479070 shutdown_connections 2023-03-27T10:15:52.693 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.206+0000 7f2b0273b700 1 --2- 172.21.15.159:0/1153479070 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2afc1061a0 0x7f2afc108580 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:52.693 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.206+0000 7f2b0273b700 1 -- 172.21.15.159:0/1153479070 >> 172.21.15.159:0/1153479070 conn(0x7f2afc0ffba0 msgr2=0x7f2afc101fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:52.693 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.206+0000 7f2b0273b700 1 -- 172.21.15.159:0/1153479070 shutdown_connections 2023-03-27T10:15:52.694 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.206+0000 7f2b0273b700 1 -- 172.21.15.159:0/1153479070 wait complete. 2023-03-27T10:15:52.694 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.206+0000 7f2b0273b700 1 Processor -- start 2023-03-27T10:15:52.694 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.206+0000 7f2b0273b700 1 -- start start 2023-03-27T10:15:52.694 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.206+0000 7f2b0273b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2afc1061a0 0x7f2afc105b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:52.694 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.206+0000 7f2b0273b700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2afc106040 con 0x7f2afc1061a0 2023-03-27T10:15:52.694 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.207+0000 7f2afbfff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2afc1061a0 0x7f2afc105b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:52.694 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.207+0000 7f2afbfff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2afc1061a0 0x7f2afc105b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44366/0 (socket says 172.21.15.159:44366) 2023-03-27T10:15:52.694 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.207+0000 7f2afbfff700 1 -- 172.21.15.159:0/3911355093 learned_addr learned my addr 172.21.15.159:0/3911355093 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:52.695 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.207+0000 7f2afbfff700 1 -- 172.21.15.159:0/3911355093 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2aec009550 con 0x7f2afc1061a0 2023-03-27T10:15:52.695 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.207+0000 7f2afbfff700 1 --2- 172.21.15.159:0/3911355093 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2afc1061a0 0x7f2afc105b00 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f2aec009130 tx=0x7f2aec011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:52.695 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.207+0000 7f2af97fa700 1 -- 172.21.15.159:0/3911355093 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f2aec003ab0 con 0x7f2afc1061a0 2023-03-27T10:15:52.695 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.207+0000 7f2af97fa700 1 -- 172.21.15.159:0/3911355093 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f2aec003c10 con 0x7f2afc1061a0 2023-03-27T10:15:52.695 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.207+0000 7f2b0273b700 1 -- 172.21.15.159:0/3911355093 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2afc109850 con 0x7f2afc1061a0 2023-03-27T10:15:52.695 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.207+0000 7f2af97fa700 1 -- 172.21.15.159:0/3911355093 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f2aec0247d0 con 0x7f2afc1061a0 2023-03-27T10:15:52.696 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.207+0000 7f2b0273b700 1 -- 172.21.15.159:0/3911355093 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2afc1096c0 con 0x7f2afc1061a0 2023-03-27T10:15:52.696 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.208+0000 7f2af97fa700 1 -- 172.21.15.159:0/3911355093 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 2) v1 ==== 43308+0+0 (secure 0 0 0) 0x7f2aec017020 con 0x7f2afc1061a0 2023-03-27T10:15:52.696 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.208+0000 7f2b0273b700 1 -- 172.21.15.159:0/3911355093 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2afc061e00 con 0x7f2afc1061a0 2023-03-27T10:15:52.696 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.208+0000 7f2af97fa700 1 -- 172.21.15.159:0/3911355093 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f2aec049630 con 0x7f2afc1061a0 2023-03-27T10:15:52.696 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.210+0000 7f2af97fa700 1 -- 172.21.15.159:0/3911355093 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72400 (secure 0 0 0) 0x7f2aec0193c0 con 0x7f2afc1061a0 2023-03-27T10:15:52.696 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.280+0000 7f2b0273b700 1 -- 172.21.15.159:0/3911355093 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7f2afc105010 con 0x7f2afc1061a0 2023-03-27T10:15:52.696 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.280+0000 7f2af97fa700 1 -- 172.21.15.159:0/3911355093 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1239 (secure 0 0 0) 0x7f2aec047020 con 0x7f2afc1061a0 2023-03-27T10:15:52.697 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.281+0000 7f2b0273b700 1 -- 172.21.15.159:0/3911355093 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2afc1061a0 msgr2=0x7f2afc105b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:52.697 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.281+0000 7f2b0273b700 1 --2- 172.21.15.159:0/3911355093 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2afc1061a0 0x7f2afc105b00 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f2aec009130 tx=0x7f2aec011ad0 comp rx=0 tx=0).stop 2023-03-27T10:15:52.697 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.281+0000 7f2b0273b700 1 -- 172.21.15.159:0/3911355093 shutdown_connections 2023-03-27T10:15:52.697 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.281+0000 7f2b0273b700 1 --2- 172.21.15.159:0/3911355093 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2afc1061a0 0x7f2afc105b00 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:52.697 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.281+0000 7f2b0273b700 1 -- 172.21.15.159:0/3911355093 >> 172.21.15.159:0/3911355093 conn(0x7f2afc0ffba0 msgr2=0x7f2afc1009c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:52.697 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.281+0000 7f2b0273b700 1 -- 172.21.15.159:0/3911355093 shutdown_connections 2023-03-27T10:15:52.697 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:52.281+0000 7f2b0273b700 1 -- 172.21.15.159:0/3911355093 wait complete. 2023-03-27T10:15:52.698 INFO:teuthology.orchestra.run.smithi159.stdout:mgr not available, waiting (3/15)... 2023-03-27T10:15:53.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:53 smithi159 ceph-mon[93039]: from='mgr.14100 172.21.15.159:0/3660332349' entity='mgr.x' 2023-03-27T10:15:53.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:53 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3911355093' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-03-27T10:15:53.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:53 smithi159 ceph-mon[93039]: mgrmap e3: x(active, since 1.00596s) 2023-03-27T10:15:55.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:55 smithi159 ceph-mon[93039]: mgrmap e4: x(active, since 2s) 2023-03-27T10:15:55.829 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 2023-03-27T10:15:55.829 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout { 2023-03-27T10:15:55.829 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "fsid": "2c3cd140-cc88-11ed-9afe-001a4aab830c", 2023-03-27T10:15:55.830 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "health": { 2023-03-27T10:15:55.830 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-03-27T10:15:55.830 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-03-27T10:15:55.830 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-03-27T10:15:55.830 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:55.830 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-03-27T10:15:55.830 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-03-27T10:15:55.830 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 0 2023-03-27T10:15:55.831 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout ], 2023-03-27T10:15:55.831 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-03-27T10:15:55.831 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "a" 2023-03-27T10:15:55.831 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout ], 2023-03-27T10:15:55.831 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "quorum_age": 15, 2023-03-27T10:15:55.831 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "monmap": { 2023-03-27T10:15:55.831 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-03-27T10:15:55.831 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2023-03-27T10:15:55.832 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-03-27T10:15:55.832 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:55.832 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-03-27T10:15:55.832 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-03-27T10:15:55.832 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-03-27T10:15:55.832 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-03-27T10:15:55.832 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-03-27T10:15:55.838 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-03-27T10:15:55.838 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-03-27T10:15:55.838 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-03-27T10:15:55.838 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:55.839 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-03-27T10:15:55.839 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-03-27T10:15:55.839 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-03-27T10:15:55.839 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-03-27T10:15:55.839 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-03-27T10:15:55.839 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-03-27T10:15:55.840 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-03-27T10:15:55.840 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-03-27T10:15:55.840 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-03-27T10:15:55.840 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:55.840 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-03-27T10:15:55.840 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-03-27T10:15:55.840 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-03-27T10:15:55.840 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-03-27T10:15:55.841 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:55.841 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-03-27T10:15:55.841 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "available": true, 2023-03-27T10:15:55.841 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-03-27T10:15:55.841 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "modules": [ 2023-03-27T10:15:55.841 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "iostat", 2023-03-27T10:15:55.841 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "nfs", 2023-03-27T10:15:55.842 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "restful" 2023-03-27T10:15:55.842 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout ], 2023-03-27T10:15:55.842 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "services": {} 2023-03-27T10:15:55.842 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:55.842 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-03-27T10:15:55.842 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-03-27T10:15:55.842 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "modified": "2023-03-27T10:15:32.617699+0000", 2023-03-27T10:15:55.842 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "services": {} 2023-03-27T10:15:55.843 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout }, 2023-03-27T10:15:55.843 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-03-27T10:15:55.843 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout } 2023-03-27T10:15:55.843 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.183+0000 7feed9047700 1 Processor -- start 2023-03-27T10:15:55.843 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.183+0000 7feed9047700 1 -- start start 2023-03-27T10:15:55.843 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.183+0000 7feed9047700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feed4104ab0 0x7feed4104ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:55.843 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.183+0000 7feed9047700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feed4105400 con 0x7feed4104ab0 2023-03-27T10:15:55.843 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.183+0000 7feed2d9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feed4104ab0 0x7feed4104ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:55.844 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.183+0000 7feed2d9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feed4104ab0 0x7feed4104ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44380/0 (socket says 172.21.15.159:44380) 2023-03-27T10:15:55.844 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.183+0000 7feed2d9d700 1 -- 172.21.15.159:0/1799827582 learned_addr learned my addr 172.21.15.159:0/1799827582 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:55.844 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.183+0000 7feed2d9d700 1 -- 172.21.15.159:0/1799827582 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feed4105540 con 0x7feed4104ab0 2023-03-27T10:15:55.844 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.184+0000 7feed2d9d700 1 --2- 172.21.15.159:0/1799827582 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feed4104ab0 0x7feed4104ec0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7feec400bb40 tx=0x7feec400be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c3ba5d3c7198423f server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:55.844 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.184+0000 7feed1d9b700 1 -- 172.21.15.159:0/1799827582 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7feec400dbe0 con 0x7feed4104ab0 2023-03-27T10:15:55.844 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.184+0000 7feed1d9b700 1 -- 172.21.15.159:0/1799827582 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7feec40044d0 con 0x7feed4104ab0 2023-03-27T10:15:55.844 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.184+0000 7feed1d9b700 1 -- 172.21.15.159:0/1799827582 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7feec4010470 con 0x7feed4104ab0 2023-03-27T10:15:55.844 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.184+0000 7feed9047700 1 -- 172.21.15.159:0/1799827582 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feed4104ab0 msgr2=0x7feed4104ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:55.845 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.184+0000 7feed9047700 1 --2- 172.21.15.159:0/1799827582 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feed4104ab0 0x7feed4104ec0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7feec400bb40 tx=0x7feec400be50 comp rx=0 tx=0).stop 2023-03-27T10:15:55.845 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.184+0000 7feed9047700 1 -- 172.21.15.159:0/1799827582 shutdown_connections 2023-03-27T10:15:55.845 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.184+0000 7feed9047700 1 --2- 172.21.15.159:0/1799827582 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feed4104ab0 0x7feed4104ec0 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:55.845 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.184+0000 7feed9047700 1 -- 172.21.15.159:0/1799827582 >> 172.21.15.159:0/1799827582 conn(0x7feed40ff3c0 msgr2=0x7feed41017d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:55.845 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.184+0000 7feed9047700 1 -- 172.21.15.159:0/1799827582 shutdown_connections 2023-03-27T10:15:55.845 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.185+0000 7feed9047700 1 -- 172.21.15.159:0/1799827582 wait complete. 2023-03-27T10:15:55.845 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.185+0000 7feed9047700 1 Processor -- start 2023-03-27T10:15:55.846 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.185+0000 7feed9047700 1 -- start start 2023-03-27T10:15:55.846 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.186+0000 7feed9047700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feed4104ab0 0x7feed4197130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:55.846 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.186+0000 7feed9047700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feed4197670 con 0x7feed4104ab0 2023-03-27T10:15:55.846 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.186+0000 7feed2d9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feed4104ab0 0x7feed4197130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:55.846 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.186+0000 7feed2d9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feed4104ab0 0x7feed4197130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44382/0 (socket says 172.21.15.159:44382) 2023-03-27T10:15:55.846 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.186+0000 7feed2d9d700 1 -- 172.21.15.159:0/3181297191 learned_addr learned my addr 172.21.15.159:0/3181297191 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:55.846 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.186+0000 7feed2d9d700 1 -- 172.21.15.159:0/3181297191 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feec400b7f0 con 0x7feed4104ab0 2023-03-27T10:15:55.846 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.186+0000 7feed2d9d700 1 --2- 172.21.15.159:0/3181297191 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feed4104ab0 0x7feed4197130 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7feec400b340 tx=0x7feec4003980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:55.847 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.186+0000 7feebbfff700 1 -- 172.21.15.159:0/3181297191 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7feec401e070 con 0x7feed4104ab0 2023-03-27T10:15:55.847 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.186+0000 7feebbfff700 1 -- 172.21.15.159:0/3181297191 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7feec40042c0 con 0x7feed4104ab0 2023-03-27T10:15:55.847 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.186+0000 7feed9047700 1 -- 172.21.15.159:0/3181297191 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feed4106670 con 0x7feed4104ab0 2023-03-27T10:15:55.847 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.186+0000 7feebbfff700 1 -- 172.21.15.159:0/3181297191 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7feec4022910 con 0x7feed4104ab0 2023-03-27T10:15:55.847 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.186+0000 7feed9047700 1 -- 172.21.15.159:0/3181297191 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feed41064e0 con 0x7feed4104ab0 2023-03-27T10:15:55.847 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.187+0000 7feebbfff700 1 -- 172.21.15.159:0/3181297191 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 4) v1 ==== 43498+0+0 (secure 0 0 0) 0x7feec4029030 con 0x7feed4104ab0 2023-03-27T10:15:55.847 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.187+0000 7feebbfff700 1 --2- 172.21.15.159:0/3181297191 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7feebc0367d0 0x7feebc038c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:55.847 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.187+0000 7feed9047700 1 -- 172.21.15.159:0/3181297191 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7feed41036b0 con 0x7feed4104ab0 2023-03-27T10:15:55.848 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.187+0000 7feebbfff700 1 -- 172.21.15.159:0/3181297191 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7feec404bba0 con 0x7feed4104ab0 2023-03-27T10:15:55.848 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.187+0000 7feed259c700 1 --2- 172.21.15.159:0/3181297191 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7feebc0367d0 0x7feebc038c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:55.848 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.188+0000 7feed259c700 1 --2- 172.21.15.159:0/3181297191 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7feebc0367d0 0x7feebc038c80 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7feec8006fd0 tx=0x7feec8006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:55.848 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.192+0000 7feebbfff700 1 -- 172.21.15.159:0/3181297191 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7feec4027080 con 0x7feed4104ab0 2023-03-27T10:15:55.848 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.450+0000 7feed9047700 1 -- 172.21.15.159:0/3181297191 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7feed402cdb0 con 0x7feed4104ab0 2023-03-27T10:15:55.848 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.450+0000 7feebbfff700 1 -- 172.21.15.159:0/3181297191 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1238 (secure 0 0 0) 0x7feec404a020 con 0x7feed4104ab0 2023-03-27T10:15:55.848 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.452+0000 7feed9047700 1 -- 172.21.15.159:0/3181297191 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7feebc0367d0 msgr2=0x7feebc038c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:55.849 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.452+0000 7feed9047700 1 --2- 172.21.15.159:0/3181297191 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7feebc0367d0 0x7feebc038c80 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7feec8006fd0 tx=0x7feec8006e40 comp rx=0 tx=0).stop 2023-03-27T10:15:55.849 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.452+0000 7feed9047700 1 -- 172.21.15.159:0/3181297191 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feed4104ab0 msgr2=0x7feed4197130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:55.849 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.452+0000 7feed9047700 1 --2- 172.21.15.159:0/3181297191 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feed4104ab0 0x7feed4197130 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7feec400b340 tx=0x7feec4003980 comp rx=0 tx=0).stop 2023-03-27T10:15:55.849 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.452+0000 7feed9047700 1 -- 172.21.15.159:0/3181297191 shutdown_connections 2023-03-27T10:15:55.849 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.452+0000 7feed9047700 1 --2- 172.21.15.159:0/3181297191 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7feebc0367d0 0x7feebc038c80 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:55.849 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.452+0000 7feed9047700 1 --2- 172.21.15.159:0/3181297191 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feed4104ab0 0x7feed4197130 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:55.849 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.452+0000 7feed9047700 1 -- 172.21.15.159:0/3181297191 >> 172.21.15.159:0/3181297191 conn(0x7feed40ff3c0 msgr2=0x7feed418e020 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:55.849 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.452+0000 7feed9047700 1 -- 172.21.15.159:0/3181297191 shutdown_connections 2023-03-27T10:15:55.850 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:55.452+0000 7feed9047700 1 -- 172.21.15.159:0/3181297191 wait complete. 2023-03-27T10:15:55.850 INFO:teuthology.orchestra.run.smithi159.stdout:mgr is available 2023-03-27T10:15:56.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:56 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3181297191' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-03-27T10:15:56.877 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 2023-03-27T10:15:56.877 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout [global] 2023-03-27T10:15:56.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout fsid = 2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:15:56.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-03-27T10:15:56.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-03-27T10:15:56.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-03-27T10:15:56.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-03-27T10:15:56.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 2023-03-27T10:15:56.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout [mgr] 2023-03-27T10:15:56.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-03-27T10:15:56.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 2023-03-27T10:15:56.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout [osd] 2023-03-27T10:15:56.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout filestore_flush_min = 0 2023-03-27T10:15:56.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mon_osd_backfillfull_ratio = 0.85 2023-03-27T10:15:56.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mon_osd_full_ratio = 0.9 2023-03-27T10:15:56.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout mon_osd_nearfull_ratio = 0.8 2023-03-27T10:15:56.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-03-27T10:15:56.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-03-27T10:15:56.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.343+0000 7f841d415700 1 Processor -- start 2023-03-27T10:15:56.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.343+0000 7f841d415700 1 -- start start 2023-03-27T10:15:56.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.343+0000 7f841d415700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8418104550 0x7f8418104960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:56.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.343+0000 7f841d415700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8418104ea0 con 0x7f8418104550 2023-03-27T10:15:56.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.344+0000 7f8416ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8418104550 0x7f8418104960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:56.881 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.344+0000 7f8416ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8418104550 0x7f8418104960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44386/0 (socket says 172.21.15.159:44386) 2023-03-27T10:15:56.881 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.344+0000 7f8416ffd700 1 -- 172.21.15.159:0/2090561998 learned_addr learned my addr 172.21.15.159:0/2090561998 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:56.881 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.344+0000 7f8416ffd700 1 -- 172.21.15.159:0/2090561998 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8418104fe0 con 0x7f8418104550 2023-03-27T10:15:56.881 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.344+0000 7f8416ffd700 1 --2- 172.21.15.159:0/2090561998 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8418104550 0x7f8418104960 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f840c00bb40 tx=0x7f840c00be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=44ae7f5da59ef693 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:56.881 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.344+0000 7f8415ffb700 1 -- 172.21.15.159:0/2090561998 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f840c00dbe0 con 0x7f8418104550 2023-03-27T10:15:56.881 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.344+0000 7f8415ffb700 1 -- 172.21.15.159:0/2090561998 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f840c0044d0 con 0x7f8418104550 2023-03-27T10:15:56.881 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.344+0000 7f8415ffb700 1 -- 172.21.15.159:0/2090561998 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f840c010470 con 0x7f8418104550 2023-03-27T10:15:56.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.345+0000 7f841d415700 1 -- 172.21.15.159:0/2090561998 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8418104550 msgr2=0x7f8418104960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:56.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.345+0000 7f841d415700 1 --2- 172.21.15.159:0/2090561998 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8418104550 0x7f8418104960 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f840c00bb40 tx=0x7f840c00be50 comp rx=0 tx=0).stop 2023-03-27T10:15:56.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.345+0000 7f841d415700 1 -- 172.21.15.159:0/2090561998 shutdown_connections 2023-03-27T10:15:56.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.345+0000 7f841d415700 1 --2- 172.21.15.159:0/2090561998 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8418104550 0x7f8418104960 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:56.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.345+0000 7f841d415700 1 -- 172.21.15.159:0/2090561998 >> 172.21.15.159:0/2090561998 conn(0x7f84180ffba0 msgr2=0x7f8418101fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:56.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.345+0000 7f841d415700 1 -- 172.21.15.159:0/2090561998 shutdown_connections 2023-03-27T10:15:56.882 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.345+0000 7f841d415700 1 -- 172.21.15.159:0/2090561998 wait complete. 2023-03-27T10:15:56.883 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.346+0000 7f841d415700 1 Processor -- start 2023-03-27T10:15:56.883 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.346+0000 7f841d415700 1 -- start start 2023-03-27T10:15:56.883 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.346+0000 7f841d415700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8418104550 0x7f8418197120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:56.883 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.346+0000 7f841d415700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8418197660 con 0x7f8418104550 2023-03-27T10:15:56.883 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.346+0000 7f8416ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8418104550 0x7f8418197120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:56.883 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.346+0000 7f8416ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8418104550 0x7f8418197120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44388/0 (socket says 172.21.15.159:44388) 2023-03-27T10:15:56.883 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.346+0000 7f8416ffd700 1 -- 172.21.15.159:0/1050136479 learned_addr learned my addr 172.21.15.159:0/1050136479 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:56.884 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.346+0000 7f8416ffd700 1 -- 172.21.15.159:0/1050136479 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f840c00b7f0 con 0x7f8418104550 2023-03-27T10:15:56.884 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.347+0000 7f8416ffd700 1 --2- 172.21.15.159:0/1050136479 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8418104550 0x7f8418197120 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f840c00d650 tx=0x7f840c004070 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:56.884 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.347+0000 7f83fffff700 1 -- 172.21.15.159:0/1050136479 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f840c0105a0 con 0x7f8418104550 2023-03-27T10:15:56.884 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.347+0000 7f83fffff700 1 -- 172.21.15.159:0/1050136479 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f840c010ba0 con 0x7f8418104550 2023-03-27T10:15:56.884 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.347+0000 7f841d415700 1 -- 172.21.15.159:0/1050136479 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f84181065a0 con 0x7f8418104550 2023-03-27T10:15:56.884 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.347+0000 7f83fffff700 1 -- 172.21.15.159:0/1050136479 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f840c021cb0 con 0x7f8418104550 2023-03-27T10:15:56.884 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.347+0000 7f841d415700 1 -- 172.21.15.159:0/1050136479 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8418106410 con 0x7f8418104550 2023-03-27T10:15:56.885 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.348+0000 7f83fffff700 1 -- 172.21.15.159:0/1050136479 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 4) v1 ==== 43498+0+0 (secure 0 0 0) 0x7f840c010d10 con 0x7f8418104550 2023-03-27T10:15:56.885 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.348+0000 7f83fffff700 1 --2- 172.21.15.159:0/1050136479 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f840003abe0 0x7f840003d090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:56.885 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.348+0000 7f83fffff700 1 -- 172.21.15.159:0/1050136479 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f840c049d80 con 0x7f8418104550 2023-03-27T10:15:56.885 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.348+0000 7f84167fc700 1 --2- 172.21.15.159:0/1050136479 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f840003abe0 0x7f840003d090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:56.885 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.348+0000 7f841d415700 1 -- 172.21.15.159:0/1050136479 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8418072e90 con 0x7f8418104550 2023-03-27T10:15:56.885 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.348+0000 7f84167fc700 1 --2- 172.21.15.159:0/1050136479 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f840003abe0 0x7f840003d090 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f8408006fd0 tx=0x7f8408006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:56.885 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.353+0000 7f83fffff700 1 -- 172.21.15.159:0/1050136479 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f840c00f5b0 con 0x7f8418104550 2023-03-27T10:15:56.886 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.533+0000 7f841d415700 1 -- 172.21.15.159:0/1050136479 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "config assimilate-conf"} v 0) v1 -- 0x7f8418190e60 con 0x7f8418104550 2023-03-27T10:15:56.886 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.534+0000 7f83fffff700 1 -- 172.21.15.159:0/1050136479 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "config assimilate-conf"}]=0 v3) v1 ==== 70+0+398 (secure 0 0 0) 0x7f840c00f3d0 con 0x7f8418104550 2023-03-27T10:15:56.886 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.536+0000 7f841d415700 1 -- 172.21.15.159:0/1050136479 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f840003abe0 msgr2=0x7f840003d090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:56.886 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.536+0000 7f841d415700 1 --2- 172.21.15.159:0/1050136479 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f840003abe0 0x7f840003d090 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f8408006fd0 tx=0x7f8408006e40 comp rx=0 tx=0).stop 2023-03-27T10:15:56.886 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.536+0000 7f841d415700 1 -- 172.21.15.159:0/1050136479 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8418104550 msgr2=0x7f8418197120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:56.886 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.536+0000 7f841d415700 1 --2- 172.21.15.159:0/1050136479 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8418104550 0x7f8418197120 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f840c00d650 tx=0x7f840c004070 comp rx=0 tx=0).stop 2023-03-27T10:15:56.886 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.536+0000 7f841d415700 1 -- 172.21.15.159:0/1050136479 shutdown_connections 2023-03-27T10:15:56.887 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.536+0000 7f841d415700 1 --2- 172.21.15.159:0/1050136479 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f840003abe0 0x7f840003d090 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:56.887 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.536+0000 7f841d415700 1 --2- 172.21.15.159:0/1050136479 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8418104550 0x7f8418197120 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:56.887 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.536+0000 7f841d415700 1 -- 172.21.15.159:0/1050136479 >> 172.21.15.159:0/1050136479 conn(0x7f84180ffba0 msgr2=0x7f841818dfa0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:56.887 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.536+0000 7f841d415700 1 -- 172.21.15.159:0/1050136479 shutdown_connections 2023-03-27T10:15:56.887 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:56.536+0000 7f841d415700 1 -- 172.21.15.159:0/1050136479 wait complete. 2023-03-27T10:15:56.887 INFO:teuthology.orchestra.run.smithi159.stdout:Enabling cephadm module... 2023-03-27T10:15:57.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:57 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/1050136479' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-03-27T10:15:58.604 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:58 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/1671514432' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-03-27T10:15:58.605 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:58 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: ignoring --setuser ceph since I am not root 2023-03-27T10:15:58.605 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:58 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: ignoring --setgroup ceph since I am not root 2023-03-27T10:15:58.606 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:58 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:58.525+0000 7f8a389a2200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-03-27T10:15:58.606 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:58 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:58.604+0000 7f8a389a2200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-03-27T10:15:58.722 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.372+0000 7f33b983b700 1 Processor -- start 2023-03-27T10:15:58.722 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.372+0000 7f33b983b700 1 -- start start 2023-03-27T10:15:58.722 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.372+0000 7f33b983b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f33b41061a0 0x7f33b4108580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:58.722 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.372+0000 7f33b983b700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f33b4073c00 con 0x7f33b41061a0 2023-03-27T10:15:58.723 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.373+0000 7f33b2ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f33b41061a0 0x7f33b4108580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:58.723 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.373+0000 7f33b2ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f33b41061a0 0x7f33b4108580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44392/0 (socket says 172.21.15.159:44392) 2023-03-27T10:15:58.723 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.373+0000 7f33b2ffd700 1 -- 172.21.15.159:0/3128978011 learned_addr learned my addr 172.21.15.159:0/3128978011 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:58.723 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.373+0000 7f33b2ffd700 1 -- 172.21.15.159:0/3128978011 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f33b4073d40 con 0x7f33b41061a0 2023-03-27T10:15:58.723 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.373+0000 7f33b2ffd700 1 --2- 172.21.15.159:0/3128978011 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f33b41061a0 0x7f33b4108580 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f33a80098a0 tx=0x7f33a8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=68b8d90c5c819521 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:58.723 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.373+0000 7f33b1ffb700 1 -- 172.21.15.159:0/3128978011 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f33a8004160 con 0x7f33b41061a0 2023-03-27T10:15:58.723 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.373+0000 7f33b1ffb700 1 -- 172.21.15.159:0/3128978011 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f33a80042c0 con 0x7f33b41061a0 2023-03-27T10:15:58.724 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.373+0000 7f33b1ffb700 1 -- 172.21.15.159:0/3128978011 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f33a8011a00 con 0x7f33b41061a0 2023-03-27T10:15:58.724 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.374+0000 7f33b983b700 1 -- 172.21.15.159:0/3128978011 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f33b41061a0 msgr2=0x7f33b4108580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:58.724 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.374+0000 7f33b983b700 1 --2- 172.21.15.159:0/3128978011 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f33b41061a0 0x7f33b4108580 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f33a80098a0 tx=0x7f33a8009bb0 comp rx=0 tx=0).stop 2023-03-27T10:15:58.724 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.374+0000 7f33b983b700 1 -- 172.21.15.159:0/3128978011 shutdown_connections 2023-03-27T10:15:58.725 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.374+0000 7f33b983b700 1 --2- 172.21.15.159:0/3128978011 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f33b41061a0 0x7f33b4108580 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:58.725 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.374+0000 7f33b983b700 1 -- 172.21.15.159:0/3128978011 >> 172.21.15.159:0/3128978011 conn(0x7f33b40ffba0 msgr2=0x7f33b4101fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:58.725 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.374+0000 7f33b983b700 1 -- 172.21.15.159:0/3128978011 shutdown_connections 2023-03-27T10:15:58.726 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.374+0000 7f33b983b700 1 -- 172.21.15.159:0/3128978011 wait complete. 2023-03-27T10:15:58.726 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.374+0000 7f33b983b700 1 Processor -- start 2023-03-27T10:15:58.726 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.375+0000 7f33b983b700 1 -- start start 2023-03-27T10:15:58.726 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.375+0000 7f33b983b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f33b41061a0 0x7f33b41992e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:58.727 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.375+0000 7f33b983b700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f33b4199820 con 0x7f33b41061a0 2023-03-27T10:15:58.727 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.375+0000 7f33b2ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f33b41061a0 0x7f33b41992e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:58.727 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.375+0000 7f33b2ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f33b41061a0 0x7f33b41992e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44394/0 (socket says 172.21.15.159:44394) 2023-03-27T10:15:58.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.375+0000 7f33b2ffd700 1 -- 172.21.15.159:0/1671514432 learned_addr learned my addr 172.21.15.159:0/1671514432 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:58.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.375+0000 7f33b2ffd700 1 -- 172.21.15.159:0/1671514432 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f33a8009550 con 0x7f33b41061a0 2023-03-27T10:15:58.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.375+0000 7f33b2ffd700 1 --2- 172.21.15.159:0/1671514432 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f33b41061a0 0x7f33b41992e0 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f33a8009130 tx=0x7f33a8011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:58.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.375+0000 7f33b8839700 1 -- 172.21.15.159:0/1671514432 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f33a8003ab0 con 0x7f33b41061a0 2023-03-27T10:15:58.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.375+0000 7f33b8839700 1 -- 172.21.15.159:0/1671514432 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f33a8003c10 con 0x7f33b41061a0 2023-03-27T10:15:58.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.375+0000 7f33b983b700 1 -- 172.21.15.159:0/1671514432 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f33b4109850 con 0x7f33b41061a0 2023-03-27T10:15:58.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.376+0000 7f33b8839700 1 -- 172.21.15.159:0/1671514432 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f33a80247d0 con 0x7f33b41061a0 2023-03-27T10:15:58.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.376+0000 7f33b983b700 1 -- 172.21.15.159:0/1671514432 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f33b41096c0 con 0x7f33b41061a0 2023-03-27T10:15:58.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.376+0000 7f33b8839700 1 -- 172.21.15.159:0/1671514432 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 4) v1 ==== 43498+0+0 (secure 0 0 0) 0x7f33a8017020 con 0x7f33b41061a0 2023-03-27T10:15:58.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.376+0000 7f33b983b700 1 -- 172.21.15.159:0/1671514432 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f33b41099e0 con 0x7f33b41061a0 2023-03-27T10:15:58.731 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.377+0000 7f33b8839700 1 --2- 172.21.15.159:0/1671514432 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f339c036820 0x7f339c038cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:58.731 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.377+0000 7f33b8839700 1 -- 172.21.15.159:0/1671514432 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f33a8049880 con 0x7f33b41061a0 2023-03-27T10:15:58.731 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.377+0000 7f33b27fc700 1 --2- 172.21.15.159:0/1671514432 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f339c036820 0x7f339c038cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:58.731 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.377+0000 7f33b27fc700 1 --2- 172.21.15.159:0/1671514432 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f339c036820 0x7f339c038cd0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f33a4006fd0 tx=0x7f33a4006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:58.732 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.381+0000 7f33b8839700 1 -- 172.21.15.159:0/1671514432 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f33a8018740 con 0x7f33b41061a0 2023-03-27T10:15:58.732 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:57.608+0000 7f33b983b700 1 -- 172.21.15.159:0/1671514432 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "cephadm"} v 0) v1 -- 0x7f33b4199e20 con 0x7f33b41061a0 2023-03-27T10:15:58.732 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:58.359+0000 7f33b8839700 1 -- 172.21.15.159:0/1671514432 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mgrmap(e 5) v1 ==== 43509+0+0 (secure 0 0 0) 0x7f33a800f050 con 0x7f33b41061a0 2023-03-27T10:15:58.732 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:58.361+0000 7f33b8839700 1 -- 172.21.15.159:0/1671514432 <== mon.0 v2:172.21.15.159:3300/0 8 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "cephadm"}]=0 v5) v1 ==== 86+0+0 (secure 0 0 0) 0x7f33a804bcb0 con 0x7f33b41061a0 2023-03-27T10:15:58.733 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:58.363+0000 7f33b983b700 1 -- 172.21.15.159:0/1671514432 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f339c036820 msgr2=0x7f339c038cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:58.733 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:58.363+0000 7f33b983b700 1 --2- 172.21.15.159:0/1671514432 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f339c036820 0x7f339c038cd0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f33a4006fd0 tx=0x7f33a4006e40 comp rx=0 tx=0).stop 2023-03-27T10:15:58.733 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:58.363+0000 7f33b983b700 1 -- 172.21.15.159:0/1671514432 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f33b41061a0 msgr2=0x7f33b41992e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:58.734 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:58.363+0000 7f33b983b700 1 --2- 172.21.15.159:0/1671514432 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f33b41061a0 0x7f33b41992e0 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f33a8009130 tx=0x7f33a8011ad0 comp rx=0 tx=0).stop 2023-03-27T10:15:58.734 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:58.363+0000 7f33b983b700 1 -- 172.21.15.159:0/1671514432 shutdown_connections 2023-03-27T10:15:58.734 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:58.363+0000 7f33b983b700 1 --2- 172.21.15.159:0/1671514432 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f339c036820 0x7f339c038cd0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:58.734 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:58.363+0000 7f33b983b700 1 --2- 172.21.15.159:0/1671514432 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f33b41061a0 0x7f33b41992e0 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:58.735 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:58.363+0000 7f33b983b700 1 -- 172.21.15.159:0/1671514432 >> 172.21.15.159:0/1671514432 conn(0x7f33b40ffba0 msgr2=0x7f33b41009c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:58.735 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:58.364+0000 7f33b983b700 1 -- 172.21.15.159:0/1671514432 shutdown_connections 2023-03-27T10:15:58.735 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:58.364+0000 7f33b983b700 1 -- 172.21.15.159:0/1671514432 wait complete. 2023-03-27T10:15:58.864 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:58 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:58.693+0000 7f8a389a2200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-03-27T10:15:58.865 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:58 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:58.794+0000 7f8a389a2200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-03-27T10:15:59.289 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:58 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:58.969+0000 7f8a389a2200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-03-27T10:15:59.575 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:59 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/1671514432' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-03-27T10:15:59.575 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:15:59 smithi159 ceph-mon[93039]: mgrmap e5: x(active, since 6s) 2023-03-27T10:15:59.576 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:59 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:59.290+0000 7f8a389a2200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-03-27T10:15:59.576 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:59 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:59.380+0000 7f8a389a2200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-03-27T10:15:59.576 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:59 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:59.513+0000 7f8a389a2200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-03-27T10:15:59.865 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout { 2023-03-27T10:15:59.865 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-03-27T10:15:59.865 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "available": true, 2023-03-27T10:15:59.866 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "active_name": "x", 2023-03-27T10:15:59.866 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-03-27T10:15:59.866 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout } 2023-03-27T10:15:59.866 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.205+0000 7f78220de700 1 Processor -- start 2023-03-27T10:15:59.866 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.205+0000 7f78220de700 1 -- start start 2023-03-27T10:15:59.866 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.205+0000 7f78220de700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f781c073bc0 0x7f781c10a5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:59.866 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.205+0000 7f78220de700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f781c105fe0 con 0x7f781c073bc0 2023-03-27T10:15:59.867 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.205+0000 7f781b7fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f781c073bc0 0x7f781c10a5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:59.867 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.205+0000 7f781b7fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f781c073bc0 0x7f781c10a5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44408/0 (socket says 172.21.15.159:44408) 2023-03-27T10:15:59.867 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.205+0000 7f781b7fe700 1 -- 172.21.15.159:0/2141429700 learned_addr learned my addr 172.21.15.159:0/2141429700 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:59.867 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.205+0000 7f781b7fe700 1 -- 172.21.15.159:0/2141429700 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f781c10ab30 con 0x7f781c073bc0 2023-03-27T10:15:59.867 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.206+0000 7f781b7fe700 1 --2- 172.21.15.159:0/2141429700 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f781c073bc0 0x7f781c10a5f0 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f780c0098a0 tx=0x7f780c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=accb53b67e9f54b0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:59.867 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.206+0000 7f781a7fc700 1 -- 172.21.15.159:0/2141429700 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f780c004160 con 0x7f781c073bc0 2023-03-27T10:15:59.867 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.206+0000 7f781a7fc700 1 -- 172.21.15.159:0/2141429700 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f780c0042c0 con 0x7f781c073bc0 2023-03-27T10:15:59.868 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.206+0000 7f781a7fc700 1 -- 172.21.15.159:0/2141429700 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f780c011a00 con 0x7f781c073bc0 2023-03-27T10:15:59.868 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.207+0000 7f78220de700 1 -- 172.21.15.159:0/2141429700 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f781c073bc0 msgr2=0x7f781c10a5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:59.868 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.207+0000 7f78220de700 1 --2- 172.21.15.159:0/2141429700 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f781c073bc0 0x7f781c10a5f0 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f780c0098a0 tx=0x7f780c009bb0 comp rx=0 tx=0).stop 2023-03-27T10:15:59.868 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.207+0000 7f78220de700 1 -- 172.21.15.159:0/2141429700 shutdown_connections 2023-03-27T10:15:59.868 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.207+0000 7f78220de700 1 --2- 172.21.15.159:0/2141429700 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f781c073bc0 0x7f781c10a5f0 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:59.868 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.207+0000 7f78220de700 1 -- 172.21.15.159:0/2141429700 >> 172.21.15.159:0/2141429700 conn(0x7f781c0ffba0 msgr2=0x7f781c101fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:59.869 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.207+0000 7f78220de700 1 -- 172.21.15.159:0/2141429700 shutdown_connections 2023-03-27T10:15:59.869 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.207+0000 7f78220de700 1 -- 172.21.15.159:0/2141429700 wait complete. 2023-03-27T10:15:59.869 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.207+0000 7f78220de700 1 Processor -- start 2023-03-27T10:15:59.869 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.208+0000 7f78220de700 1 -- start start 2023-03-27T10:15:59.869 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.208+0000 7f78220de700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f781c19b550 0x7f781c19b960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:59.869 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.208+0000 7f78220de700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f781c19bea0 con 0x7f781c19b550 2023-03-27T10:15:59.869 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.208+0000 7f781b7fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f781c19b550 0x7f781c19b960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:15:59.870 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.208+0000 7f781b7fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f781c19b550 0x7f781c19b960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44410/0 (socket says 172.21.15.159:44410) 2023-03-27T10:15:59.870 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.208+0000 7f781b7fe700 1 -- 172.21.15.159:0/2442281820 learned_addr learned my addr 172.21.15.159:0/2442281820 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:15:59.870 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.208+0000 7f781b7fe700 1 -- 172.21.15.159:0/2442281820 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f780c009550 con 0x7f781c19b550 2023-03-27T10:15:59.870 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.208+0000 7f781b7fe700 1 --2- 172.21.15.159:0/2442281820 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f781c19b550 0x7f781c19b960 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f780c009130 tx=0x7f780c011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:15:59.870 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.208+0000 7f7818ff9700 1 -- 172.21.15.159:0/2442281820 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f780c003ab0 con 0x7f781c19b550 2023-03-27T10:15:59.870 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.208+0000 7f7818ff9700 1 -- 172.21.15.159:0/2442281820 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f780c003c10 con 0x7f781c19b550 2023-03-27T10:15:59.870 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.208+0000 7f78220de700 1 -- 172.21.15.159:0/2442281820 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f781c10bac0 con 0x7f781c19b550 2023-03-27T10:15:59.870 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.208+0000 7f7818ff9700 1 -- 172.21.15.159:0/2442281820 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f780c0247d0 con 0x7f781c19b550 2023-03-27T10:15:59.871 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.208+0000 7f78220de700 1 -- 172.21.15.159:0/2442281820 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f781c10b930 con 0x7f781c19b550 2023-03-27T10:15:59.871 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.209+0000 7f7818ff9700 1 -- 172.21.15.159:0/2442281820 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 5) v1 ==== 43509+0+0 (secure 0 0 0) 0x7f780c017020 con 0x7f781c19b550 2023-03-27T10:15:59.871 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.209+0000 7f78220de700 1 -- 172.21.15.159:0/2442281820 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f781c061e00 con 0x7f781c19b550 2023-03-27T10:15:59.871 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.209+0000 7f7818ff9700 1 --2- 172.21.15.159:0/2442281820 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7804036820 0x7f7804038cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:15:59.871 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.209+0000 7f7818ff9700 1 -- 172.21.15.159:0/2442281820 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f780c049990 con 0x7f781c19b550 2023-03-27T10:15:59.871 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.209+0000 7f781affd700 1 -- 172.21.15.159:0/2442281820 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7804036820 msgr2=0x7f7804038cd0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.159:6800/1729835725 2023-03-27T10:15:59.871 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.210+0000 7f781affd700 1 --2- 172.21.15.159:0/2442281820 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7804036820 0x7f7804038cd0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-03-27T10:15:59.872 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.214+0000 7f7818ff9700 1 -- 172.21.15.159:0/2442281820 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f780c02de30 con 0x7f781c19b550 2023-03-27T10:15:59.872 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.411+0000 7f781affd700 1 -- 172.21.15.159:0/2442281820 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7804036820 msgr2=0x7f7804038cd0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.159:6800/1729835725 2023-03-27T10:15:59.872 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.411+0000 7f781affd700 1 --2- 172.21.15.159:0/2442281820 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7804036820 0x7f7804038cd0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2023-03-27T10:15:59.872 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.453+0000 7f78220de700 1 -- 172.21.15.159:0/2442281820 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "mgr stat"} v 0) v1 -- 0x7f781c073bc0 con 0x7f781c19b550 2023-03-27T10:15:59.872 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.453+0000 7f7818ff9700 1 -- 172.21.15.159:0/2442281820 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "mgr stat"}]=0 v5) v1 ==== 56+0+88 (secure 0 0 0) 0x7f780c0172b0 con 0x7f781c19b550 2023-03-27T10:15:59.872 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.455+0000 7f78220de700 1 -- 172.21.15.159:0/2442281820 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7804036820 msgr2=0x7f7804038cd0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-03-27T10:15:59.873 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.455+0000 7f78220de700 1 --2- 172.21.15.159:0/2442281820 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7804036820 0x7f7804038cd0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:59.873 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.455+0000 7f78220de700 1 -- 172.21.15.159:0/2442281820 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f781c19b550 msgr2=0x7f781c19b960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:15:59.873 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.455+0000 7f78220de700 1 --2- 172.21.15.159:0/2442281820 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f781c19b550 0x7f781c19b960 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f780c009130 tx=0x7f780c011ad0 comp rx=0 tx=0).stop 2023-03-27T10:15:59.873 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.455+0000 7f78220de700 1 -- 172.21.15.159:0/2442281820 shutdown_connections 2023-03-27T10:15:59.873 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.455+0000 7f78220de700 1 --2- 172.21.15.159:0/2442281820 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7804036820 0x7f7804038cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:59.873 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.455+0000 7f78220de700 1 --2- 172.21.15.159:0/2442281820 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f781c19b550 0x7f781c19b960 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:15:59.873 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.455+0000 7f78220de700 1 -- 172.21.15.159:0/2442281820 >> 172.21.15.159:0/2442281820 conn(0x7f781c0ffba0 msgr2=0x7f781c101270 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:15:59.874 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.456+0000 7f78220de700 1 -- 172.21.15.159:0/2442281820 shutdown_connections 2023-03-27T10:15:59.874 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:15:59.456+0000 7f78220de700 1 -- 172.21.15.159:0/2442281820 wait complete. 2023-03-27T10:15:59.874 INFO:teuthology.orchestra.run.smithi159.stdout:Waiting for the mgr to restart... 2023-03-27T10:15:59.874 INFO:teuthology.orchestra.run.smithi159.stdout:Waiting for mgr epoch 5... 2023-03-27T10:15:59.874 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:15:59 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:15:59.614+0000 7f8a389a2200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-03-27T10:16:00.359 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:00 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:00.048+0000 7f8a389a2200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-03-27T10:16:00.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:00 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/2442281820' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-03-27T10:16:01.114 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:00 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:00.634+0000 7f8a389a2200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-03-27T10:16:01.115 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:00 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:00.802+0000 7f8a389a2200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-03-27T10:16:01.806 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:01 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:01.551+0000 7f8a389a2200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-03-27T10:16:01.806 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:01 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:01.640+0000 7f8a389a2200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-03-27T10:16:01.806 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:01 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:01.806+0000 7f8a389a2200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-03-27T10:16:02.114 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:01 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:01.991+0000 7f8a389a2200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-03-27T10:16:02.531 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:02 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:02.137+0000 7f8a389a2200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-03-27T10:16:02.864 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:02 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:02.532+0000 7f8a389a2200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-03-27T10:16:03.362 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:03 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:03.057+0000 7f8a389a2200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-03-27T10:16:03.614 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:03 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:03.362+0000 7f8a389a2200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-03-27T10:16:03.615 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:03 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:03.550+0000 7f8a389a2200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-03-27T10:16:04.114 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:03 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:03.662+0000 7f8a389a2200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-03-27T10:16:04.115 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:03 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:03.873+0000 7f8a389a2200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-03-27T10:16:05.114 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:04 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:04.723+0000 7f8a389a2200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-03-27T10:16:05.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:04 smithi159 ceph-mon[93039]: Active manager daemon x restarted 2023-03-27T10:16:05.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:04 smithi159 ceph-mon[93039]: Activating manager daemon x 2023-03-27T10:16:05.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:04 smithi159 ceph-mon[93039]: osdmap e2: 0 total, 0 up, 0 in 2023-03-27T10:16:06.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: mgrmap e6: x(active, starting, since 0.0526707s) 2023-03-27T10:16:06.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:16:06.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-03-27T10:16:06.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-03-27T10:16:06.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-03-27T10:16:06.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-03-27T10:16:06.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: Manager daemon x is now available 2023-03-27T10:16:06.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: Found migration_current of "None". Setting to last migration. 2023-03-27T10:16:06.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' 2023-03-27T10:16:06.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' 2023-03-27T10:16:06.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:16:06.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:16:06.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:16:06.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-03-27T10:16:06.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-03-27T10:16:06.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' 2023-03-27T10:16:06.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:05 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' 2023-03-27T10:16:06.122 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout { 2023-03-27T10:16:06.122 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-03-27T10:16:06.123 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "initialized": true 2023-03-27T10:16:06.123 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout } 2023-03-27T10:16:06.123 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.548+0000 7f702dda5700 1 Processor -- start 2023-03-27T10:16:06.123 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.549+0000 7f702dda5700 1 -- start start 2023-03-27T10:16:06.123 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.549+0000 7f702dda5700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7028103f80 0x7f7028104390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:06.123 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.549+0000 7f702dda5700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7028073c00 con 0x7f7028103f80 2023-03-27T10:16:06.123 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.549+0000 7f70277fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7028103f80 0x7f7028104390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:06.124 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.549+0000 7f70277fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7028103f80 0x7f7028104390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44418/0 (socket says 172.21.15.159:44418) 2023-03-27T10:16:06.124 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.549+0000 7f70277fe700 1 -- 172.21.15.159:0/2984159225 learned_addr learned my addr 172.21.15.159:0/2984159225 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:06.124 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.549+0000 7f70277fe700 1 -- 172.21.15.159:0/2984159225 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7028073d40 con 0x7f7028103f80 2023-03-27T10:16:06.124 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.549+0000 7f70277fe700 1 --2- 172.21.15.159:0/2984159225 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7028103f80 0x7f7028104390 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f701c0098a0 tx=0x7f701c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2a8efd0378ce9c77 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:06.124 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.550+0000 7f70267fc700 1 -- 172.21.15.159:0/2984159225 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f701c004160 con 0x7f7028103f80 2023-03-27T10:16:06.124 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.550+0000 7f70267fc700 1 -- 172.21.15.159:0/2984159225 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f701c0042c0 con 0x7f7028103f80 2023-03-27T10:16:06.124 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.550+0000 7f70267fc700 1 -- 172.21.15.159:0/2984159225 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f701c011a00 con 0x7f7028103f80 2023-03-27T10:16:06.124 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.550+0000 7f702dda5700 1 -- 172.21.15.159:0/2984159225 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7028103f80 msgr2=0x7f7028104390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:06.125 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.550+0000 7f702dda5700 1 --2- 172.21.15.159:0/2984159225 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7028103f80 0x7f7028104390 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f701c0098a0 tx=0x7f701c009bb0 comp rx=0 tx=0).stop 2023-03-27T10:16:06.125 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.550+0000 7f702dda5700 1 -- 172.21.15.159:0/2984159225 shutdown_connections 2023-03-27T10:16:06.125 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.550+0000 7f702dda5700 1 --2- 172.21.15.159:0/2984159225 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7028103f80 0x7f7028104390 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:06.125 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.550+0000 7f702dda5700 1 -- 172.21.15.159:0/2984159225 >> 172.21.15.159:0/2984159225 conn(0x7f70280ffb90 msgr2=0x7f7028101fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:06.125 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.550+0000 7f702dda5700 1 -- 172.21.15.159:0/2984159225 shutdown_connections 2023-03-27T10:16:06.125 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.550+0000 7f702dda5700 1 -- 172.21.15.159:0/2984159225 wait complete. 2023-03-27T10:16:06.125 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.551+0000 7f702dda5700 1 Processor -- start 2023-03-27T10:16:06.126 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.551+0000 7f702dda5700 1 -- start start 2023-03-27T10:16:06.126 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.552+0000 7f702dda5700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7028103f80 0x7f702819b510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:06.126 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.552+0000 7f702dda5700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f702819ba50 con 0x7f7028103f80 2023-03-27T10:16:06.126 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.552+0000 7f70277fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7028103f80 0x7f702819b510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:06.126 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.552+0000 7f70277fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7028103f80 0x7f702819b510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44420/0 (socket says 172.21.15.159:44420) 2023-03-27T10:16:06.126 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.552+0000 7f70277fe700 1 -- 172.21.15.159:0/2381411503 learned_addr learned my addr 172.21.15.159:0/2381411503 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:06.126 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.552+0000 7f70277fe700 1 -- 172.21.15.159:0/2381411503 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f701c009550 con 0x7f7028103f80 2023-03-27T10:16:06.126 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.552+0000 7f70277fe700 1 --2- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7028103f80 0x7f702819b510 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f701c009130 tx=0x7f701c011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:06.127 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.552+0000 7f7024ff9700 1 -- 172.21.15.159:0/2381411503 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f701c003ab0 con 0x7f7028103f80 2023-03-27T10:16:06.127 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.552+0000 7f7024ff9700 1 -- 172.21.15.159:0/2381411503 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f701c003c10 con 0x7f7028103f80 2023-03-27T10:16:06.127 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.552+0000 7f7024ff9700 1 -- 172.21.15.159:0/2381411503 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f701c0247d0 con 0x7f7028103f80 2023-03-27T10:16:06.127 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.552+0000 7f702dda5700 1 -- 172.21.15.159:0/2381411503 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7028105660 con 0x7f7028103f80 2023-03-27T10:16:06.127 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.552+0000 7f702dda5700 1 -- 172.21.15.159:0/2381411503 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f70281054d0 con 0x7f7028103f80 2023-03-27T10:16:06.127 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.553+0000 7f7024ff9700 1 -- 172.21.15.159:0/2381411503 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 5) v1 ==== 43509+0+0 (secure 0 0 0) 0x7f701c017020 con 0x7f7028103f80 2023-03-27T10:16:06.128 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.553+0000 7f7024ff9700 1 --2- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7010036870 0x7f7010038d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:06.128 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.553+0000 7f702dda5700 1 -- 172.21.15.159:0/2381411503 --> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7f70281057f0 con 0x7f7010036870 2023-03-27T10:16:06.128 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.553+0000 7f7024ff9700 1 -- 172.21.15.159:0/2381411503 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f701c049900 con 0x7f7028103f80 2023-03-27T10:16:06.128 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.553+0000 7f7026ffd700 1 -- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7010036870 msgr2=0x7f7010038d20 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.159:6800/1729835725 2023-03-27T10:16:06.128 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.553+0000 7f7026ffd700 1 --2- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7010036870 0x7f7010038d20 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-03-27T10:16:06.128 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.754+0000 7f7026ffd700 1 -- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7010036870 msgr2=0x7f7010038d20 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.159:6800/1729835725 2023-03-27T10:16:06.128 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:00.754+0000 7f7026ffd700 1 --2- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7010036870 0x7f7010038d20 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2023-03-27T10:16:06.128 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:01.155+0000 7f7026ffd700 1 -- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7010036870 msgr2=0x7f7010038d20 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.159:6800/1729835725 2023-03-27T10:16:06.129 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:01.155+0000 7f7026ffd700 1 --2- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7010036870 0x7f7010038d20 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2023-03-27T10:16:06.129 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:01.956+0000 7f7026ffd700 1 -- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7010036870 msgr2=0x7f7010038d20 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.159:6800/1729835725 2023-03-27T10:16:06.129 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:01.956+0000 7f7026ffd700 1 --2- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7010036870 0x7f7010038d20 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2023-03-27T10:16:06.129 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:03.557+0000 7f7026ffd700 1 -- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7010036870 msgr2=0x7f7010038d20 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.159:6800/1729835725 2023-03-27T10:16:06.129 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:03.557+0000 7f7026ffd700 1 --2- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7010036870 0x7f7010038d20 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 3.200000 2023-03-27T10:16:06.129 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:04.787+0000 7f7024ff9700 1 -- 172.21.15.159:0/2381411503 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mgrmap(e 6) v1 ==== 43319+0+0 (secure 0 0 0) 0x7f701c00f050 con 0x7f7028103f80 2023-03-27T10:16:06.129 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:04.788+0000 7f7024ff9700 1 -- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7010036870 msgr2=0x7f7010038d20 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-03-27T10:16:06.130 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:04.788+0000 7f7024ff9700 1 --2- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7010036870 0x7f7010038d20 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:06.130 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.797+0000 7f7024ff9700 1 -- 172.21.15.159:0/2381411503 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mgrmap(e 7) v1 ==== 43429+0+0 (secure 0 0 0) 0x7f701c04af40 con 0x7f7028103f80 2023-03-27T10:16:06.130 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.797+0000 7f7024ff9700 1 --2- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f7010039fa0 0x7f701003c380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:06.130 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.797+0000 7f7024ff9700 1 -- 172.21.15.159:0/2381411503 --> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7f70281057f0 con 0x7f7010039fa0 2023-03-27T10:16:06.130 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.798+0000 7f7026ffd700 1 --2- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f7010039fa0 0x7f701003c380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:06.130 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.798+0000 7f7026ffd700 1 --2- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f7010039fa0 0x7f701003c380 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7f7018003a10 tx=0x7f70180092b0 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:06.130 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.799+0000 7f7024ff9700 1 -- 172.21.15.159:0/2381411503 <== mgr.14120 v2:172.21.15.159:6800/3561797228 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+6549 (secure 0 0 0) 0x7f70281057f0 con 0x7f7010039fa0 2023-03-27T10:16:06.130 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.805+0000 7f702dda5700 1 -- 172.21.15.159:0/2381411503 --> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] -- command(tid 1: {"prefix": "mgr_status"}) v1 -- 0x7f702819c850 con 0x7f7010039fa0 2023-03-27T10:16:06.131 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.805+0000 7f7024ff9700 1 -- 172.21.15.159:0/2381411503 <== mgr.14120 v2:172.21.15.159:6800/3561797228 2 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+51 (secure 0 0 0) 0x7f702819c850 con 0x7f7010039fa0 2023-03-27T10:16:06.131 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.805+0000 7f702dda5700 1 -- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f7010039fa0 msgr2=0x7f701003c380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:06.131 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.805+0000 7f702dda5700 1 --2- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f7010039fa0 0x7f701003c380 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7f7018003a10 tx=0x7f70180092b0 comp rx=0 tx=0).stop 2023-03-27T10:16:06.131 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.805+0000 7f702dda5700 1 -- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7028103f80 msgr2=0x7f702819b510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:06.131 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.805+0000 7f702dda5700 1 --2- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7028103f80 0x7f702819b510 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f701c009130 tx=0x7f701c011ad0 comp rx=0 tx=0).stop 2023-03-27T10:16:06.131 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.805+0000 7f702dda5700 1 -- 172.21.15.159:0/2381411503 shutdown_connections 2023-03-27T10:16:06.131 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.805+0000 7f702dda5700 1 --2- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7028103f80 0x7f702819b510 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:06.131 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.805+0000 7f702dda5700 1 --2- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f7010039fa0 0x7f701003c380 unknown :-1 s=CLOSED pgs=3 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:06.132 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.805+0000 7f702dda5700 1 --2- 172.21.15.159:0/2381411503 >> [v2:172.21.15.159:6800/1729835725,v1:172.21.15.159:6801/1729835725] conn(0x7f7010036870 0x7f7010038d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:06.132 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.805+0000 7f702dda5700 1 -- 172.21.15.159:0/2381411503 >> 172.21.15.159:0/2381411503 conn(0x7f70280ffb90 msgr2=0x7f702818fb20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:06.132 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.805+0000 7f702dda5700 1 -- 172.21.15.159:0/2381411503 shutdown_connections 2023-03-27T10:16:06.132 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:05.805+0000 7f702dda5700 1 -- 172.21.15.159:0/2381411503 wait complete. 2023-03-27T10:16:06.132 INFO:teuthology.orchestra.run.smithi159.stdout:mgr epoch 5 is available 2023-03-27T10:16:06.132 INFO:teuthology.orchestra.run.smithi159.stdout:Setting orchestrator backend to cephadm... 2023-03-27T10:16:07.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:06 smithi159 ceph-mon[93039]: mgrmap e7: x(active, since 1.06267s) 2023-03-27T10:16:07.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:06 smithi159 ceph-mon[93039]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-03-27T10:16:07.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:06 smithi159 ceph-mon[93039]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-03-27T10:16:07.161 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.593+0000 7fd8a13c0700 1 Processor -- start 2023-03-27T10:16:07.162 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.593+0000 7fd8a13c0700 1 -- start start 2023-03-27T10:16:07.162 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.594+0000 7fd8a13c0700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd89c104550 0x7fd89c104960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:07.162 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.594+0000 7fd8a13c0700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd89c104ea0 con 0x7fd89c104550 2023-03-27T10:16:07.162 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.594+0000 7fd89affd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd89c104550 0x7fd89c104960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:07.162 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.594+0000 7fd89affd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd89c104550 0x7fd89c104960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44464/0 (socket says 172.21.15.159:44464) 2023-03-27T10:16:07.162 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.594+0000 7fd89affd700 1 -- 172.21.15.159:0/8911481 learned_addr learned my addr 172.21.15.159:0/8911481 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:07.163 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.594+0000 7fd89affd700 1 -- 172.21.15.159:0/8911481 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd89c104fe0 con 0x7fd89c104550 2023-03-27T10:16:07.163 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.594+0000 7fd89affd700 1 --2- 172.21.15.159:0/8911481 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd89c104550 0x7fd89c104960 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fd88c00bb40 tx=0x7fd88c00be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=85c482a8b9ac52f6 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:07.163 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.594+0000 7fd899ffb700 1 -- 172.21.15.159:0/8911481 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd88c00dbe0 con 0x7fd89c104550 2023-03-27T10:16:07.163 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.594+0000 7fd899ffb700 1 -- 172.21.15.159:0/8911481 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fd88c0044d0 con 0x7fd89c104550 2023-03-27T10:16:07.163 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.594+0000 7fd899ffb700 1 -- 172.21.15.159:0/8911481 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd88c010470 con 0x7fd89c104550 2023-03-27T10:16:07.163 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.595+0000 7fd8a13c0700 1 -- 172.21.15.159:0/8911481 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd89c104550 msgr2=0x7fd89c104960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:07.164 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.595+0000 7fd8a13c0700 1 --2- 172.21.15.159:0/8911481 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd89c104550 0x7fd89c104960 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fd88c00bb40 tx=0x7fd88c00be50 comp rx=0 tx=0).stop 2023-03-27T10:16:07.164 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.595+0000 7fd8a13c0700 1 -- 172.21.15.159:0/8911481 shutdown_connections 2023-03-27T10:16:07.164 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.595+0000 7fd8a13c0700 1 --2- 172.21.15.159:0/8911481 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd89c104550 0x7fd89c104960 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:07.164 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.595+0000 7fd8a13c0700 1 -- 172.21.15.159:0/8911481 >> 172.21.15.159:0/8911481 conn(0x7fd89c0ffba0 msgr2=0x7fd89c101fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:07.164 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.595+0000 7fd8a13c0700 1 -- 172.21.15.159:0/8911481 shutdown_connections 2023-03-27T10:16:07.164 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.595+0000 7fd8a13c0700 1 -- 172.21.15.159:0/8911481 wait complete. 2023-03-27T10:16:07.164 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.596+0000 7fd8a13c0700 1 Processor -- start 2023-03-27T10:16:07.164 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.596+0000 7fd8a13c0700 1 -- start start 2023-03-27T10:16:07.165 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.596+0000 7fd8a13c0700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd89c104550 0x7fd89c197120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:07.165 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.596+0000 7fd8a13c0700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd89c197660 con 0x7fd89c104550 2023-03-27T10:16:07.165 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.596+0000 7fd89affd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd89c104550 0x7fd89c197120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:07.165 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.596+0000 7fd89affd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd89c104550 0x7fd89c197120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44466/0 (socket says 172.21.15.159:44466) 2023-03-27T10:16:07.165 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.596+0000 7fd89affd700 1 -- 172.21.15.159:0/1490811735 learned_addr learned my addr 172.21.15.159:0/1490811735 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:07.165 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.596+0000 7fd89affd700 1 -- 172.21.15.159:0/1490811735 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd88c00b7f0 con 0x7fd89c104550 2023-03-27T10:16:07.165 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.597+0000 7fd89affd700 1 --2- 172.21.15.159:0/1490811735 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd89c104550 0x7fd89c197120 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7fd88c003b20 tx=0x7fd88c003980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:07.166 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.597+0000 7fd883fff700 1 -- 172.21.15.159:0/1490811735 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd88c01e070 con 0x7fd89c104550 2023-03-27T10:16:07.166 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.597+0000 7fd883fff700 1 -- 172.21.15.159:0/1490811735 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fd88c004300 con 0x7fd89c104550 2023-03-27T10:16:07.166 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.597+0000 7fd8a13c0700 1 -- 172.21.15.159:0/1490811735 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd89c1065a0 con 0x7fd89c104550 2023-03-27T10:16:07.166 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.597+0000 7fd883fff700 1 -- 172.21.15.159:0/1490811735 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd88c022a50 con 0x7fd89c104550 2023-03-27T10:16:07.166 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.597+0000 7fd8a13c0700 1 -- 172.21.15.159:0/1490811735 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd89c106410 con 0x7fd89c104550 2023-03-27T10:16:07.166 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.598+0000 7fd883fff700 1 -- 172.21.15.159:0/1490811735 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 7) v1 ==== 43429+0+0 (secure 0 0 0) 0x7fd88c029030 con 0x7fd89c104550 2023-03-27T10:16:07.166 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.598+0000 7fd8a13c0700 1 -- 172.21.15.159:0/1490811735 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd89c072e90 con 0x7fd89c104550 2023-03-27T10:16:07.166 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.598+0000 7fd883fff700 1 --2- 172.21.15.159:0/1490811735 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7fd8840367e0 0x7fd884038c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:07.167 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.598+0000 7fd883fff700 1 -- 172.21.15.159:0/1490811735 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7fd88c04c2d0 con 0x7fd89c104550 2023-03-27T10:16:07.167 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.598+0000 7fd89a7fc700 1 --2- 172.21.15.159:0/1490811735 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7fd8840367e0 0x7fd884038c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:07.167 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.599+0000 7fd89a7fc700 1 --2- 172.21.15.159:0/1490811735 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7fd8840367e0 0x7fd884038c90 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fd890006fd0 tx=0x7fd890006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:07.167 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.603+0000 7fd883fff700 1 -- 172.21.15.159:0/1490811735 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fd88c0106e0 con 0x7fd89c104550 2023-03-27T10:16:07.167 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.801+0000 7fd883fff700 1 -- 172.21.15.159:0/1490811735 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mgrmap(e 8) v1 ==== 43509+0+0 (secure 0 0 0) 0x7fd88c0108c0 con 0x7fd89c104550 2023-03-27T10:16:07.167 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.806+0000 7fd8a13c0700 1 -- 172.21.15.159:0/1490811735 --> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] -- mgr_command(tid 0: {"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}) v1 -- 0x7fd89c060c10 con 0x7fd8840367e0 2023-03-27T10:16:07.167 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.812+0000 7fd883fff700 1 -- 172.21.15.159:0/1490811735 <== mgr.14120 v2:172.21.15.159:6800/3561797228 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7fd89c060c10 con 0x7fd8840367e0 2023-03-27T10:16:07.168 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.815+0000 7fd8a13c0700 1 -- 172.21.15.159:0/1490811735 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7fd8840367e0 msgr2=0x7fd884038c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:07.168 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.815+0000 7fd8a13c0700 1 --2- 172.21.15.159:0/1490811735 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7fd8840367e0 0x7fd884038c90 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fd890006fd0 tx=0x7fd890006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:07.168 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.815+0000 7fd8a13c0700 1 -- 172.21.15.159:0/1490811735 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd89c104550 msgr2=0x7fd89c197120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:07.168 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.815+0000 7fd8a13c0700 1 --2- 172.21.15.159:0/1490811735 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd89c104550 0x7fd89c197120 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7fd88c003b20 tx=0x7fd88c003980 comp rx=0 tx=0).stop 2023-03-27T10:16:07.168 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.815+0000 7fd8a13c0700 1 -- 172.21.15.159:0/1490811735 shutdown_connections 2023-03-27T10:16:07.168 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.815+0000 7fd8a13c0700 1 --2- 172.21.15.159:0/1490811735 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7fd8840367e0 0x7fd884038c90 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:07.168 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.815+0000 7fd8a13c0700 1 --2- 172.21.15.159:0/1490811735 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd89c104550 0x7fd89c197120 secure :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7fd88c003b20 tx=0x7fd88c003980 comp rx=0 tx=0).stop 2023-03-27T10:16:07.168 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.815+0000 7fd8a13c0700 1 -- 172.21.15.159:0/1490811735 >> 172.21.15.159:0/1490811735 conn(0x7fd89c0ffba0 msgr2=0x7fd89c18dfa0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:07.169 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.815+0000 7fd8a13c0700 1 -- 172.21.15.159:0/1490811735 shutdown_connections 2023-03-27T10:16:07.169 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:06.815+0000 7fd8a13c0700 1 -- 172.21.15.159:0/1490811735 wait complete. 2023-03-27T10:16:08.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:07 smithi159 ceph-mon[93039]: mgrmap e8: x(active, since 2s) 2023-03-27T10:16:08.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:07 smithi159 ceph-mon[93039]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:16:08.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:07 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' 2023-03-27T10:16:08.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:07 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:16:08.205 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout value unchanged 2023-03-27T10:16:08.205 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.603+0000 7f45ca31e700 1 Processor -- start 2023-03-27T10:16:08.206 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.603+0000 7f45ca31e700 1 -- start start 2023-03-27T10:16:08.206 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.604+0000 7f45ca31e700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f45c4104550 0x7f45c4104960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:08.206 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.604+0000 7f45ca31e700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f45c4104ea0 con 0x7f45c4104550 2023-03-27T10:16:08.206 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.604+0000 7f45c3fff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f45c4104550 0x7f45c4104960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:08.206 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.604+0000 7f45c3fff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f45c4104550 0x7f45c4104960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44470/0 (socket says 172.21.15.159:44470) 2023-03-27T10:16:08.206 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.604+0000 7f45c3fff700 1 -- 172.21.15.159:0/74853121 learned_addr learned my addr 172.21.15.159:0/74853121 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:08.207 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.604+0000 7f45c3fff700 1 -- 172.21.15.159:0/74853121 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f45c4104fe0 con 0x7f45c4104550 2023-03-27T10:16:08.207 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.604+0000 7f45c3fff700 1 --2- 172.21.15.159:0/74853121 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f45c4104550 0x7f45c4104960 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f45b40098a0 tx=0x7f45b4009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8cddc7d4b55fdcf6 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:08.207 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.604+0000 7f45c2ffd700 1 -- 172.21.15.159:0/74853121 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f45b4004160 con 0x7f45c4104550 2023-03-27T10:16:08.207 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.604+0000 7f45c2ffd700 1 -- 172.21.15.159:0/74853121 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f45b40042c0 con 0x7f45c4104550 2023-03-27T10:16:08.207 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.604+0000 7f45c2ffd700 1 -- 172.21.15.159:0/74853121 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f45b4011a00 con 0x7f45c4104550 2023-03-27T10:16:08.207 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.605+0000 7f45ca31e700 1 -- 172.21.15.159:0/74853121 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f45c4104550 msgr2=0x7f45c4104960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:08.207 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.605+0000 7f45ca31e700 1 --2- 172.21.15.159:0/74853121 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f45c4104550 0x7f45c4104960 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f45b40098a0 tx=0x7f45b4009bb0 comp rx=0 tx=0).stop 2023-03-27T10:16:08.207 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.605+0000 7f45ca31e700 1 -- 172.21.15.159:0/74853121 shutdown_connections 2023-03-27T10:16:08.208 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.605+0000 7f45ca31e700 1 --2- 172.21.15.159:0/74853121 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f45c4104550 0x7f45c4104960 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:08.208 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.605+0000 7f45ca31e700 1 -- 172.21.15.159:0/74853121 >> 172.21.15.159:0/74853121 conn(0x7f45c40ffba0 msgr2=0x7f45c4101fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:08.208 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.605+0000 7f45ca31e700 1 -- 172.21.15.159:0/74853121 shutdown_connections 2023-03-27T10:16:08.208 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.605+0000 7f45ca31e700 1 -- 172.21.15.159:0/74853121 wait complete. 2023-03-27T10:16:08.208 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.606+0000 7f45ca31e700 1 Processor -- start 2023-03-27T10:16:08.208 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.606+0000 7f45ca31e700 1 -- start start 2023-03-27T10:16:08.209 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.606+0000 7f45ca31e700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f45c4104550 0x7f45c4198270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:08.209 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.606+0000 7f45ca31e700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f45c41987b0 con 0x7f45c4104550 2023-03-27T10:16:08.209 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.606+0000 7f45c3fff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f45c4104550 0x7f45c4198270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:08.209 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.606+0000 7f45c3fff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f45c4104550 0x7f45c4198270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44472/0 (socket says 172.21.15.159:44472) 2023-03-27T10:16:08.209 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.606+0000 7f45c3fff700 1 -- 172.21.15.159:0/3887076204 learned_addr learned my addr 172.21.15.159:0/3887076204 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:08.209 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.607+0000 7f45c3fff700 1 -- 172.21.15.159:0/3887076204 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f45b4009550 con 0x7f45c4104550 2023-03-27T10:16:08.209 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.607+0000 7f45c3fff700 1 --2- 172.21.15.159:0/3887076204 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f45c4104550 0x7f45c4198270 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f45b4009fb0 tx=0x7f45b4011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:08.209 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.607+0000 7f45c17fa700 1 -- 172.21.15.159:0/3887076204 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f45b4003a40 con 0x7f45c4104550 2023-03-27T10:16:08.210 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.607+0000 7f45c17fa700 1 -- 172.21.15.159:0/3887076204 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f45b4003ba0 con 0x7f45c4104550 2023-03-27T10:16:08.210 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.607+0000 7f45ca31e700 1 -- 172.21.15.159:0/3887076204 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f45c41065a0 con 0x7f45c4104550 2023-03-27T10:16:08.210 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.607+0000 7f45c17fa700 1 -- 172.21.15.159:0/3887076204 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f45b401a750 con 0x7f45c4104550 2023-03-27T10:16:08.210 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.607+0000 7f45ca31e700 1 -- 172.21.15.159:0/3887076204 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f45c4106410 con 0x7f45c4104550 2023-03-27T10:16:08.210 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.608+0000 7f45c17fa700 1 -- 172.21.15.159:0/3887076204 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 8) v1 ==== 43509+0+0 (secure 0 0 0) 0x7f45b4017020 con 0x7f45c4104550 2023-03-27T10:16:08.210 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.608+0000 7f45c17fa700 1 --2- 172.21.15.159:0/3887076204 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f45ac036820 0x7f45ac038cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:08.210 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.608+0000 7f45c17fa700 1 -- 172.21.15.159:0/3887076204 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f45b4049b80 con 0x7f45c4104550 2023-03-27T10:16:08.211 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.608+0000 7f45c37fe700 1 --2- 172.21.15.159:0/3887076204 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f45ac036820 0x7f45ac038cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:08.211 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.608+0000 7f45ca31e700 1 -- 172.21.15.159:0/3887076204 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f45c4108a20 con 0x7f45c4104550 2023-03-27T10:16:08.211 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.609+0000 7f45c37fe700 1 --2- 172.21.15.159:0/3887076204 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f45ac036820 0x7f45ac038cd0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f45b8006fd0 tx=0x7f45b8006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:08.211 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.613+0000 7f45c17fa700 1 -- 172.21.15.159:0/3887076204 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f45b4049e60 con 0x7f45c4104550 2023-03-27T10:16:08.211 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.803+0000 7f45ca31e700 1 -- 172.21.15.159:0/3887076204 --> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] -- mgr_command(tid 0: {"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}) v1 -- 0x7f45c4101030 con 0x7f45ac036820 2023-03-27T10:16:08.211 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.804+0000 7f45c17fa700 1 -- 172.21.15.159:0/3887076204 <== mgr.14120 v2:172.21.15.159:6800/3561797228 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+16 (secure 0 0 0) 0x7f45c4101030 con 0x7f45ac036820 2023-03-27T10:16:08.211 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.806+0000 7f45ca31e700 1 -- 172.21.15.159:0/3887076204 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f45ac036820 msgr2=0x7f45ac038cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:08.211 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.806+0000 7f45ca31e700 1 --2- 172.21.15.159:0/3887076204 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f45ac036820 0x7f45ac038cd0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f45b8006fd0 tx=0x7f45b8006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:08.212 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.806+0000 7f45ca31e700 1 -- 172.21.15.159:0/3887076204 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f45c4104550 msgr2=0x7f45c4198270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:08.212 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.806+0000 7f45ca31e700 1 --2- 172.21.15.159:0/3887076204 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f45c4104550 0x7f45c4198270 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f45b4009fb0 tx=0x7f45b4011ad0 comp rx=0 tx=0).stop 2023-03-27T10:16:08.212 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.806+0000 7f45ca31e700 1 -- 172.21.15.159:0/3887076204 shutdown_connections 2023-03-27T10:16:08.212 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.806+0000 7f45ca31e700 1 --2- 172.21.15.159:0/3887076204 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f45ac036820 0x7f45ac038cd0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:08.212 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.806+0000 7f45ca31e700 1 --2- 172.21.15.159:0/3887076204 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f45c4104550 0x7f45c4198270 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:08.212 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.806+0000 7f45ca31e700 1 -- 172.21.15.159:0/3887076204 >> 172.21.15.159:0/3887076204 conn(0x7f45c40ffba0 msgr2=0x7f45c4100920 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:08.212 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.806+0000 7f45ca31e700 1 -- 172.21.15.159:0/3887076204 shutdown_connections 2023-03-27T10:16:08.213 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:07.806+0000 7f45ca31e700 1 -- 172.21.15.159:0/3887076204 wait complete. 2023-03-27T10:16:08.213 INFO:teuthology.orchestra.run.smithi159.stdout:Generating ssh key... 2023-03-27T10:16:09.247 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.619+0000 7ff6470c5700 1 Processor -- start 2023-03-27T10:16:09.247 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.619+0000 7ff6470c5700 1 -- start start 2023-03-27T10:16:09.247 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.619+0000 7ff6470c5700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff640105c60 0x7ff640106070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:09.248 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.619+0000 7ff6470c5700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff6401065b0 con 0x7ff640105c60 2023-03-27T10:16:09.248 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.620+0000 7ff644e61700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff640105c60 0x7ff640106070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:09.248 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.620+0000 7ff644e61700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff640105c60 0x7ff640106070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44478/0 (socket says 172.21.15.159:44478) 2023-03-27T10:16:09.248 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.620+0000 7ff644e61700 1 -- 172.21.15.159:0/612578449 learned_addr learned my addr 172.21.15.159:0/612578449 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:09.248 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.620+0000 7ff644e61700 1 -- 172.21.15.159:0/612578449 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff6401066f0 con 0x7ff640105c60 2023-03-27T10:16:09.248 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.620+0000 7ff644e61700 1 --2- 172.21.15.159:0/612578449 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff640105c60 0x7ff640106070 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7ff6340098a0 tx=0x7ff634009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=39eb6408b23c4450 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:09.248 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.620+0000 7ff63f7fe700 1 -- 172.21.15.159:0/612578449 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7ff634004160 con 0x7ff640105c60 2023-03-27T10:16:09.249 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.620+0000 7ff63f7fe700 1 -- 172.21.15.159:0/612578449 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7ff6340042c0 con 0x7ff640105c60 2023-03-27T10:16:09.249 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.620+0000 7ff63f7fe700 1 -- 172.21.15.159:0/612578449 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7ff634011a00 con 0x7ff640105c60 2023-03-27T10:16:09.249 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.621+0000 7ff6470c5700 1 -- 172.21.15.159:0/612578449 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff640105c60 msgr2=0x7ff640106070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:09.249 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.621+0000 7ff6470c5700 1 --2- 172.21.15.159:0/612578449 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff640105c60 0x7ff640106070 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7ff6340098a0 tx=0x7ff634009bb0 comp rx=0 tx=0).stop 2023-03-27T10:16:09.249 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.621+0000 7ff6470c5700 1 -- 172.21.15.159:0/612578449 shutdown_connections 2023-03-27T10:16:09.249 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.621+0000 7ff6470c5700 1 --2- 172.21.15.159:0/612578449 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff640105c60 0x7ff640106070 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:09.249 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.621+0000 7ff6470c5700 1 -- 172.21.15.159:0/612578449 >> 172.21.15.159:0/612578449 conn(0x7ff6400ff3c0 msgr2=0x7ff6401017d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:09.249 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.621+0000 7ff6470c5700 1 -- 172.21.15.159:0/612578449 shutdown_connections 2023-03-27T10:16:09.250 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.621+0000 7ff6470c5700 1 -- 172.21.15.159:0/612578449 wait complete. 2023-03-27T10:16:09.250 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.622+0000 7ff6470c5700 1 Processor -- start 2023-03-27T10:16:09.250 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.622+0000 7ff6470c5700 1 -- start start 2023-03-27T10:16:09.250 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.622+0000 7ff6470c5700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff640105c60 0x7ff64019b4d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:09.250 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.622+0000 7ff6470c5700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff64019ba10 con 0x7ff640105c60 2023-03-27T10:16:09.250 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.622+0000 7ff644e61700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff640105c60 0x7ff64019b4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:09.250 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.622+0000 7ff644e61700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff640105c60 0x7ff64019b4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44480/0 (socket says 172.21.15.159:44480) 2023-03-27T10:16:09.251 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.622+0000 7ff644e61700 1 -- 172.21.15.159:0/3801745093 learned_addr learned my addr 172.21.15.159:0/3801745093 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:09.251 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.622+0000 7ff644e61700 1 -- 172.21.15.159:0/3801745093 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff634009550 con 0x7ff640105c60 2023-03-27T10:16:09.251 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.622+0000 7ff644e61700 1 --2- 172.21.15.159:0/3801745093 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff640105c60 0x7ff64019b4d0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7ff634009fb0 tx=0x7ff634011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:09.251 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.622+0000 7ff63dffb700 1 -- 172.21.15.159:0/3801745093 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7ff634003930 con 0x7ff640105c60 2023-03-27T10:16:09.251 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.623+0000 7ff63dffb700 1 -- 172.21.15.159:0/3801745093 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7ff634003a90 con 0x7ff640105c60 2023-03-27T10:16:09.251 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.623+0000 7ff6470c5700 1 -- 172.21.15.159:0/3801745093 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff64010bbb0 con 0x7ff640105c60 2023-03-27T10:16:09.252 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.623+0000 7ff63dffb700 1 -- 172.21.15.159:0/3801745093 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7ff634019610 con 0x7ff640105c60 2023-03-27T10:16:09.252 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.623+0000 7ff6470c5700 1 -- 172.21.15.159:0/3801745093 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff64010ba20 con 0x7ff640105c60 2023-03-27T10:16:09.252 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.624+0000 7ff63dffb700 1 -- 172.21.15.159:0/3801745093 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 8) v1 ==== 43509+0+0 (secure 0 0 0) 0x7ff63402a020 con 0x7ff640105c60 2023-03-27T10:16:09.252 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.624+0000 7ff63dffb700 1 --2- 172.21.15.159:0/3801745093 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7ff630036820 0x7ff630038cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:09.252 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.624+0000 7ff6470c5700 1 -- 172.21.15.159:0/3801745093 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff64010bd40 con 0x7ff640105c60 2023-03-27T10:16:09.252 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.624+0000 7ff63dffb700 1 -- 172.21.15.159:0/3801745093 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7ff634049e10 con 0x7ff640105c60 2023-03-27T10:16:09.252 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.624+0000 7ff63ffff700 1 --2- 172.21.15.159:0/3801745093 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7ff630036820 0x7ff630038cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:09.253 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.624+0000 7ff63ffff700 1 --2- 172.21.15.159:0/3801745093 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7ff630036820 0x7ff630038cd0 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7ff62c006fd0 tx=0x7ff62c006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:09.253 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.629+0000 7ff63dffb700 1 -- 172.21.15.159:0/3801745093 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7ff63401a430 con 0x7ff640105c60 2023-03-27T10:16:09.253 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.814+0000 7ff6470c5700 1 -- 172.21.15.159:0/3801745093 --> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] -- mgr_command(tid 0: {"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}) v1 -- 0x7ff640100800 con 0x7ff630036820 2023-03-27T10:16:09.253 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.947+0000 7ff63dffb700 1 -- 172.21.15.159:0/3801745093 <== mgr.14120 v2:172.21.15.159:6800/3561797228 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7ff640100800 con 0x7ff630036820 2023-03-27T10:16:09.253 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.949+0000 7ff6470c5700 1 -- 172.21.15.159:0/3801745093 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7ff630036820 msgr2=0x7ff630038cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:09.253 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.949+0000 7ff6470c5700 1 --2- 172.21.15.159:0/3801745093 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7ff630036820 0x7ff630038cd0 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7ff62c006fd0 tx=0x7ff62c006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:09.253 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.949+0000 7ff6470c5700 1 -- 172.21.15.159:0/3801745093 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff640105c60 msgr2=0x7ff64019b4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:09.253 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.949+0000 7ff6470c5700 1 --2- 172.21.15.159:0/3801745093 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff640105c60 0x7ff64019b4d0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7ff634009fb0 tx=0x7ff634011ad0 comp rx=0 tx=0).stop 2023-03-27T10:16:09.254 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.949+0000 7ff6470c5700 1 -- 172.21.15.159:0/3801745093 shutdown_connections 2023-03-27T10:16:09.254 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.949+0000 7ff6470c5700 1 --2- 172.21.15.159:0/3801745093 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7ff630036820 0x7ff630038cd0 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:09.254 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.949+0000 7ff6470c5700 1 --2- 172.21.15.159:0/3801745093 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff640105c60 0x7ff64019b4d0 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:09.254 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.949+0000 7ff6470c5700 1 -- 172.21.15.159:0/3801745093 >> 172.21.15.159:0/3801745093 conn(0x7ff6400ff3c0 msgr2=0x7ff6401000f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:09.254 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.950+0000 7ff6470c5700 1 -- 172.21.15.159:0/3801745093 shutdown_connections 2023-03-27T10:16:09.254 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:08.950+0000 7ff6470c5700 1 -- 172.21.15.159:0/3801745093 wait complete. 2023-03-27T10:16:09.275 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: Generating public/private rsa key pair. 2023-03-27T10:16:09.275 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: Your identification has been saved in /tmp/tmp0_fnfc8v/key. 2023-03-27T10:16:09.275 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: Your public key has been saved in /tmp/tmp0_fnfc8v/key.pub. 2023-03-27T10:16:09.275 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: The key fingerprint is: 2023-03-27T10:16:09.276 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: SHA256:lMTvtVuE+pN7a4k7EIv8w77XHLi66v4P06xeosWB3sU ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:16:09.276 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: The key's randomart image is: 2023-03-27T10:16:09.276 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: +---[RSA 3072]----+ 2023-03-27T10:16:09.276 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: | .. | 2023-03-27T10:16:09.276 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: | ... | 2023-03-27T10:16:09.276 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: | o. . | 2023-03-27T10:16:09.276 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: | . .o.o . | 2023-03-27T10:16:09.277 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: | .So.=E+ | 2023-03-27T10:16:09.277 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: | .oo==o o | 2023-03-27T10:16:09.277 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: | .oBo+O o | 2023-03-27T10:16:09.277 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: | o+*X * | 2023-03-27T10:16:09.277 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: | o==OB+B.. | 2023-03-27T10:16:09.277 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: +----[SHA256]-----+ 2023-03-27T10:16:09.278 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-mon[93039]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:16:09.278 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' 2023-03-27T10:16:09.278 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' 2023-03-27T10:16:09.278 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:08 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:16:10.282 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:09 smithi159 ceph-mon[93039]: [27/Mar/2023:10:16:08] ENGINE Bus STARTING 2023-03-27T10:16:10.282 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:09 smithi159 ceph-mon[93039]: [27/Mar/2023:10:16:08] ENGINE Serving on https://172.21.15.159:8765 2023-03-27T10:16:10.282 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:09 smithi159 ceph-mon[93039]: [27/Mar/2023:10:16:08] ENGINE Serving on https://172.21.15.159:7150 2023-03-27T10:16:10.282 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:09 smithi159 ceph-mon[93039]: [27/Mar/2023:10:16:08] ENGINE Bus STARTED 2023-03-27T10:16:10.283 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:09 smithi159 ceph-mon[93039]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:16:10.283 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:09 smithi159 ceph-mon[93039]: Generating ssh key... 2023-03-27T10:16:10.283 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:09 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' 2023-03-27T10:16:10.283 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:09 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' 2023-03-27T10:16:10.291 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx1sQLTEmyAgXsE9t6c87JfjiJetYiuI+ur7emKZG6fw1dcawIFy68YejeEc52YSsc93Ez/ySp37R2DbAdyLlqFqiEyKL6VVT+gzzKenY3MpodS7C4NpWthSxkK8HlOGxzWklb8T5GhI+ZB3xb3Yp08Jej8lGQf7xPldxpjlmdOpnKdFQqmuQBgeLEljN3cHlfzd9KHSBrF1g0VgBsX6cRUE2Kx8rmOFpsVn7VDH9u4vDHb3poS1V19NXOxoLsvHh1O6PJsxLgwNwtPOdXxVoIiXqXL81cOjI7UoMoisq3IwcsvWIV7a2Ngbg0nxP/D0+TjyTpyP7aWt4esL6AyKZiMr0snTCqW8HGzLwJ7fDH+4Gama60NlUs497uzJ44o/DP6XBozhn/H2l5+pTBKDNCoRiInszpw7kLXxGunmEH7azHmWljTaIId80HVaN/fWZgGY/xZRE6JNqeZNfX2uNSY4KG4C/FqEcc9hBxfuDCdi9cNQk+k7vE9MmNHNwwx2s= ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:16:10.291 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.731+0000 7f2888a6e700 1 Processor -- start 2023-03-27T10:16:10.291 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.732+0000 7f2888a6e700 1 -- start start 2023-03-27T10:16:10.291 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.732+0000 7f2888a6e700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f288407ac90 0x7f2884079190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:10.292 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.732+0000 7f2888a6e700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28840796d0 con 0x7f288407ac90 2023-03-27T10:16:10.292 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.732+0000 7f288259c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f288407ac90 0x7f2884079190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:10.292 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.732+0000 7f288259c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f288407ac90 0x7f2884079190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44490/0 (socket says 172.21.15.159:44490) 2023-03-27T10:16:10.292 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.732+0000 7f288259c700 1 -- 172.21.15.159:0/1914326734 learned_addr learned my addr 172.21.15.159:0/1914326734 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:10.292 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.732+0000 7f288259c700 1 -- 172.21.15.159:0/1914326734 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2884079810 con 0x7f288407ac90 2023-03-27T10:16:10.292 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.733+0000 7f288259c700 1 --2- 172.21.15.159:0/1914326734 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f288407ac90 0x7f2884079190 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f286c0098a0 tx=0x7f286c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8509067c92efe97e server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:10.292 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.733+0000 7f288159a700 1 -- 172.21.15.159:0/1914326734 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f286c004160 con 0x7f288407ac90 2023-03-27T10:16:10.292 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.733+0000 7f288159a700 1 -- 172.21.15.159:0/1914326734 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f286c0042c0 con 0x7f288407ac90 2023-03-27T10:16:10.293 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.733+0000 7f288159a700 1 -- 172.21.15.159:0/1914326734 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f286c011a00 con 0x7f288407ac90 2023-03-27T10:16:10.293 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.733+0000 7f2888a6e700 1 -- 172.21.15.159:0/1914326734 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f288407ac90 msgr2=0x7f2884079190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:10.293 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.733+0000 7f2888a6e700 1 --2- 172.21.15.159:0/1914326734 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f288407ac90 0x7f2884079190 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f286c0098a0 tx=0x7f286c009bb0 comp rx=0 tx=0).stop 2023-03-27T10:16:10.293 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.733+0000 7f2888a6e700 1 -- 172.21.15.159:0/1914326734 shutdown_connections 2023-03-27T10:16:10.293 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.733+0000 7f2888a6e700 1 --2- 172.21.15.159:0/1914326734 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f288407ac90 0x7f2884079190 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:10.293 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.733+0000 7f2888a6e700 1 -- 172.21.15.159:0/1914326734 >> 172.21.15.159:0/1914326734 conn(0x7f2884100d20 msgr2=0x7f2884103130 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:10.293 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.733+0000 7f2888a6e700 1 -- 172.21.15.159:0/1914326734 shutdown_connections 2023-03-27T10:16:10.294 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.733+0000 7f2888a6e700 1 -- 172.21.15.159:0/1914326734 wait complete. 2023-03-27T10:16:10.294 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.734+0000 7f2888a6e700 1 Processor -- start 2023-03-27T10:16:10.294 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.734+0000 7f2888a6e700 1 -- start start 2023-03-27T10:16:10.294 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.734+0000 7f2888a6e700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f288407ac90 0x7f288419b500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:10.294 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.734+0000 7f2888a6e700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f288419ba40 con 0x7f288407ac90 2023-03-27T10:16:10.294 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.734+0000 7f288259c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f288407ac90 0x7f288419b500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:10.294 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.734+0000 7f288259c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f288407ac90 0x7f288419b500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44492/0 (socket says 172.21.15.159:44492) 2023-03-27T10:16:10.294 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.734+0000 7f288259c700 1 -- 172.21.15.159:0/2880631165 learned_addr learned my addr 172.21.15.159:0/2880631165 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:10.295 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.735+0000 7f288259c700 1 -- 172.21.15.159:0/2880631165 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f286c009550 con 0x7f288407ac90 2023-03-27T10:16:10.295 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.735+0000 7f288259c700 1 --2- 172.21.15.159:0/2880631165 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f288407ac90 0x7f288419b500 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f286c006af0 tx=0x7f286c011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:10.295 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.735+0000 7f287b7fe700 1 -- 172.21.15.159:0/2880631165 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f286c003a40 con 0x7f288407ac90 2023-03-27T10:16:10.295 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.735+0000 7f287b7fe700 1 -- 172.21.15.159:0/2880631165 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f286c003ba0 con 0x7f288407ac90 2023-03-27T10:16:10.295 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.735+0000 7f2888a6e700 1 -- 172.21.15.159:0/2880631165 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f288410bb60 con 0x7f288407ac90 2023-03-27T10:16:10.295 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.735+0000 7f287b7fe700 1 -- 172.21.15.159:0/2880631165 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f286c01a700 con 0x7f288407ac90 2023-03-27T10:16:10.295 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.735+0000 7f2888a6e700 1 -- 172.21.15.159:0/2880631165 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f288410b9d0 con 0x7f288407ac90 2023-03-27T10:16:10.296 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.736+0000 7f287b7fe700 1 -- 172.21.15.159:0/2880631165 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 8) v1 ==== 43509+0+0 (secure 0 0 0) 0x7f286c017020 con 0x7f288407ac90 2023-03-27T10:16:10.296 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.736+0000 7f287b7fe700 1 --2- 172.21.15.159:0/2880631165 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f2870036820 0x7f2870038cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:10.296 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.736+0000 7f2888a6e700 1 -- 172.21.15.159:0/2880631165 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f288410bcf0 con 0x7f288407ac90 2023-03-27T10:16:10.296 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.736+0000 7f287b7fe700 1 -- 172.21.15.159:0/2880631165 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f286c049b90 con 0x7f288407ac90 2023-03-27T10:16:10.296 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.736+0000 7f2881d9b700 1 --2- 172.21.15.159:0/2880631165 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f2870036820 0x7f2870038cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:10.296 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.737+0000 7f2881d9b700 1 --2- 172.21.15.159:0/2880631165 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f2870036820 0x7f2870038cd0 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f2874006fd0 tx=0x7f2874006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:10.296 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.741+0000 7f287b7fe700 1 -- 172.21.15.159:0/2880631165 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f286c018db0 con 0x7f288407ac90 2023-03-27T10:16:10.296 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.924+0000 7f2888a6e700 1 -- 172.21.15.159:0/2880631165 --> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] -- mgr_command(tid 0: {"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}) v1 -- 0x7f2884102160 con 0x7f2870036820 2023-03-27T10:16:10.297 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.925+0000 7f287b7fe700 1 -- 172.21.15.159:0/2880631165 <== mgr.14120 v2:172.21.15.159:6800/3561797228 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+595 (secure 0 0 0) 0x7f2884102160 con 0x7f2870036820 2023-03-27T10:16:10.297 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.927+0000 7f2888a6e700 1 -- 172.21.15.159:0/2880631165 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f2870036820 msgr2=0x7f2870038cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:10.297 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.927+0000 7f2888a6e700 1 --2- 172.21.15.159:0/2880631165 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f2870036820 0x7f2870038cd0 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f2874006fd0 tx=0x7f2874006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:10.297 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.927+0000 7f2888a6e700 1 -- 172.21.15.159:0/2880631165 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f288407ac90 msgr2=0x7f288419b500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:10.297 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.927+0000 7f2888a6e700 1 --2- 172.21.15.159:0/2880631165 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f288407ac90 0x7f288419b500 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f286c006af0 tx=0x7f286c011ad0 comp rx=0 tx=0).stop 2023-03-27T10:16:10.297 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.927+0000 7f2888a6e700 1 -- 172.21.15.159:0/2880631165 shutdown_connections 2023-03-27T10:16:10.297 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.927+0000 7f2888a6e700 1 --2- 172.21.15.159:0/2880631165 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f2870036820 0x7f2870038cd0 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:10.297 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.927+0000 7f2888a6e700 1 --2- 172.21.15.159:0/2880631165 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f288407ac90 0x7f288419b500 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:10.298 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.927+0000 7f2888a6e700 1 -- 172.21.15.159:0/2880631165 >> 172.21.15.159:0/2880631165 conn(0x7f2884100d20 msgr2=0x7f2884101a50 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:10.298 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.927+0000 7f2888a6e700 1 -- 172.21.15.159:0/2880631165 shutdown_connections 2023-03-27T10:16:10.298 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:09.927+0000 7f2888a6e700 1 -- 172.21.15.159:0/2880631165 wait complete. 2023-03-27T10:16:10.298 INFO:teuthology.orchestra.run.smithi159.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-03-27T10:16:10.298 INFO:teuthology.orchestra.run.smithi159.stdout:Adding key to root@localhost authorized_keys... 2023-03-27T10:16:10.298 INFO:teuthology.orchestra.run.smithi159.stdout:Adding host smithi159... 2023-03-27T10:16:11.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:11 smithi159 ceph-mon[93039]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:16:12.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:12 smithi159 ceph-mon[93039]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi159", "addr": "172.21.15.159", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:16:12.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:12 smithi159 ceph-mon[93039]: Deploying cephadm binary to smithi159 2023-03-27T10:16:13.743 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout Added host 'smithi159' with addr '172.21.15.159' 2023-03-27T10:16:13.743 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.274+0000 7f997ced8700 1 Processor -- start 2023-03-27T10:16:13.743 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.274+0000 7f997ced8700 1 -- start start 2023-03-27T10:16:13.743 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.274+0000 7f997ced8700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9978106190 0x7f9978108570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:13.743 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.274+0000 7f997ced8700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9978073c00 con 0x7f9978106190 2023-03-27T10:16:13.744 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.275+0000 7f997659c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9978106190 0x7f9978108570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:13.744 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.275+0000 7f997659c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9978106190 0x7f9978108570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44496/0 (socket says 172.21.15.159:44496) 2023-03-27T10:16:13.744 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.275+0000 7f997659c700 1 -- 172.21.15.159:0/3706403399 learned_addr learned my addr 172.21.15.159:0/3706403399 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:13.744 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.275+0000 7f997659c700 1 -- 172.21.15.159:0/3706403399 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9978073d40 con 0x7f9978106190 2023-03-27T10:16:13.744 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.275+0000 7f997659c700 1 --2- 172.21.15.159:0/3706403399 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9978106190 0x7f9978108570 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f996c0098a0 tx=0x7f996c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=701c113644ed987f server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:13.744 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.275+0000 7f997559a700 1 -- 172.21.15.159:0/3706403399 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f996c004160 con 0x7f9978106190 2023-03-27T10:16:13.744 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.275+0000 7f997559a700 1 -- 172.21.15.159:0/3706403399 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f996c0042c0 con 0x7f9978106190 2023-03-27T10:16:13.745 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.275+0000 7f997559a700 1 -- 172.21.15.159:0/3706403399 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f996c011a00 con 0x7f9978106190 2023-03-27T10:16:13.745 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.276+0000 7f997ced8700 1 -- 172.21.15.159:0/3706403399 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9978106190 msgr2=0x7f9978108570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:13.745 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.276+0000 7f997ced8700 1 --2- 172.21.15.159:0/3706403399 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9978106190 0x7f9978108570 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f996c0098a0 tx=0x7f996c009bb0 comp rx=0 tx=0).stop 2023-03-27T10:16:13.745 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.276+0000 7f997ced8700 1 -- 172.21.15.159:0/3706403399 shutdown_connections 2023-03-27T10:16:13.745 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.276+0000 7f997ced8700 1 --2- 172.21.15.159:0/3706403399 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9978106190 0x7f9978108570 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:13.745 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.276+0000 7f997ced8700 1 -- 172.21.15.159:0/3706403399 >> 172.21.15.159:0/3706403399 conn(0x7f99780ffb90 msgr2=0x7f9978101fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:13.745 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.276+0000 7f997ced8700 1 -- 172.21.15.159:0/3706403399 shutdown_connections 2023-03-27T10:16:13.746 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.276+0000 7f997ced8700 1 -- 172.21.15.159:0/3706403399 wait complete. 2023-03-27T10:16:13.746 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.277+0000 7f997ced8700 1 Processor -- start 2023-03-27T10:16:13.746 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.277+0000 7f997ced8700 1 -- start start 2023-03-27T10:16:13.746 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.277+0000 7f997ced8700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9978106190 0x7f99781992d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:13.746 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.277+0000 7f997ced8700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9978199810 con 0x7f9978106190 2023-03-27T10:16:13.746 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.277+0000 7f997659c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9978106190 0x7f99781992d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:13.746 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.277+0000 7f997659c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9978106190 0x7f99781992d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44498/0 (socket says 172.21.15.159:44498) 2023-03-27T10:16:13.746 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.277+0000 7f997659c700 1 -- 172.21.15.159:0/326592452 learned_addr learned my addr 172.21.15.159:0/326592452 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:13.747 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.278+0000 7f997659c700 1 -- 172.21.15.159:0/326592452 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f996c009550 con 0x7f9978106190 2023-03-27T10:16:13.747 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.278+0000 7f997659c700 1 --2- 172.21.15.159:0/326592452 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9978106190 0x7f99781992d0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f996c009fb0 tx=0x7f996c011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:13.747 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.278+0000 7f99677fe700 1 -- 172.21.15.159:0/326592452 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f996c003a40 con 0x7f9978106190 2023-03-27T10:16:13.747 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.278+0000 7f99677fe700 1 -- 172.21.15.159:0/326592452 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f996c003ba0 con 0x7f9978106190 2023-03-27T10:16:13.747 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.278+0000 7f997ced8700 1 -- 172.21.15.159:0/326592452 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9978109840 con 0x7f9978106190 2023-03-27T10:16:13.747 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.278+0000 7f99677fe700 1 -- 172.21.15.159:0/326592452 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f996c01a750 con 0x7f9978106190 2023-03-27T10:16:13.747 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.278+0000 7f997ced8700 1 -- 172.21.15.159:0/326592452 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f99781096b0 con 0x7f9978106190 2023-03-27T10:16:13.747 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.279+0000 7f99677fe700 1 -- 172.21.15.159:0/326592452 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 8) v1 ==== 43509+0+0 (secure 0 0 0) 0x7f996c017020 con 0x7f9978106190 2023-03-27T10:16:13.748 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.279+0000 7f99677fe700 1 --2- 172.21.15.159:0/326592452 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f99600367d0 0x7f9960038c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:13.748 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.279+0000 7f99677fe700 1 -- 172.21.15.159:0/326592452 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f996c049b20 con 0x7f9978106190 2023-03-27T10:16:13.748 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.279+0000 7f997ced8700 1 -- 172.21.15.159:0/326592452 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f99781099d0 con 0x7f9978106190 2023-03-27T10:16:13.748 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.279+0000 7f9975d9b700 1 --2- 172.21.15.159:0/326592452 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f99600367d0 0x7f9960038c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:13.748 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.279+0000 7f9975d9b700 1 --2- 172.21.15.159:0/326592452 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f99600367d0 0x7f9960038c80 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f9968006fd0 tx=0x7f9968006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:13.748 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.283+0000 7f99677fe700 1 -- 172.21.15.159:0/326592452 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f996c018db0 con 0x7f9978106190 2023-03-27T10:16:13.749 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:11.471+0000 7f997ced8700 1 -- 172.21.15.159:0/326592452 --> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi159", "addr": "172.21.15.159", "target": ["mon-mgr", ""]}) v1 -- 0x7f9978101080 con 0x7f99600367d0 2023-03-27T10:16:13.749 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:13.381+0000 7f99677fe700 1 -- 172.21.15.159:0/326592452 <== mgr.14120 v2:172.21.15.159:6800/3561797228 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+49 (secure 0 0 0) 0x7f9978101080 con 0x7f99600367d0 2023-03-27T10:16:13.749 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:13.384+0000 7f997ced8700 1 -- 172.21.15.159:0/326592452 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f99600367d0 msgr2=0x7f9960038c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:13.749 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:13.384+0000 7f997ced8700 1 --2- 172.21.15.159:0/326592452 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f99600367d0 0x7f9960038c80 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f9968006fd0 tx=0x7f9968006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:13.749 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:13.384+0000 7f997ced8700 1 -- 172.21.15.159:0/326592452 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9978106190 msgr2=0x7f99781992d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:13.749 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:13.384+0000 7f997ced8700 1 --2- 172.21.15.159:0/326592452 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9978106190 0x7f99781992d0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f996c009fb0 tx=0x7f996c011ad0 comp rx=0 tx=0).stop 2023-03-27T10:16:13.749 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:13.384+0000 7f997ced8700 1 -- 172.21.15.159:0/326592452 shutdown_connections 2023-03-27T10:16:13.750 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:13.384+0000 7f997ced8700 1 --2- 172.21.15.159:0/326592452 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f99600367d0 0x7f9960038c80 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:13.750 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:13.384+0000 7f997ced8700 1 --2- 172.21.15.159:0/326592452 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9978106190 0x7f99781992d0 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:13.750 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:13.384+0000 7f997ced8700 1 -- 172.21.15.159:0/326592452 >> 172.21.15.159:0/326592452 conn(0x7f99780ffb90 msgr2=0x7f9978100970 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:13.750 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:13.384+0000 7f997ced8700 1 -- 172.21.15.159:0/326592452 shutdown_connections 2023-03-27T10:16:13.750 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:13.384+0000 7f997ced8700 1 -- 172.21.15.159:0/326592452 wait complete. 2023-03-27T10:16:13.750 INFO:teuthology.orchestra.run.smithi159.stdout:Deploying unmanaged mon service... 2023-03-27T10:16:14.527 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:14 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' 2023-03-27T10:16:14.527 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:14 smithi159 ceph-mon[93039]: Added host smithi159 2023-03-27T10:16:14.527 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:14 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:16:14.987 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-03-27T10:16:14.988 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.185+0000 7f0c6910a700 1 Processor -- start 2023-03-27T10:16:14.988 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.185+0000 7f0c6910a700 1 -- start start 2023-03-27T10:16:14.988 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.185+0000 7f0c6910a700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0c64106160 0x7f0c64108540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:14.988 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.185+0000 7f0c6910a700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c64073c40 con 0x7f0c64106160 2023-03-27T10:16:14.988 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.185+0000 7f0c62d9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0c64106160 0x7f0c64108540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:14.988 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.185+0000 7f0c62d9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0c64106160 0x7f0c64108540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44522/0 (socket says 172.21.15.159:44522) 2023-03-27T10:16:14.989 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.185+0000 7f0c62d9d700 1 -- 172.21.15.159:0/2033899784 learned_addr learned my addr 172.21.15.159:0/2033899784 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:14.989 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.185+0000 7f0c62d9d700 1 -- 172.21.15.159:0/2033899784 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0c64073d80 con 0x7f0c64106160 2023-03-27T10:16:14.989 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.185+0000 7f0c62d9d700 1 --2- 172.21.15.159:0/2033899784 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0c64106160 0x7f0c64108540 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f0c540098a0 tx=0x7f0c54009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9d6311ea30d09a45 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:14.989 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.186+0000 7f0c61d9b700 1 -- 172.21.15.159:0/2033899784 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f0c54004160 con 0x7f0c64106160 2023-03-27T10:16:14.989 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.186+0000 7f0c61d9b700 1 -- 172.21.15.159:0/2033899784 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f0c540042c0 con 0x7f0c64106160 2023-03-27T10:16:14.989 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.186+0000 7f0c61d9b700 1 -- 172.21.15.159:0/2033899784 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f0c54011a00 con 0x7f0c64106160 2023-03-27T10:16:14.989 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.186+0000 7f0c6910a700 1 -- 172.21.15.159:0/2033899784 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0c64106160 msgr2=0x7f0c64108540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:14.990 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.186+0000 7f0c6910a700 1 --2- 172.21.15.159:0/2033899784 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0c64106160 0x7f0c64108540 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f0c540098a0 tx=0x7f0c54009bb0 comp rx=0 tx=0).stop 2023-03-27T10:16:14.990 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.186+0000 7f0c6910a700 1 -- 172.21.15.159:0/2033899784 shutdown_connections 2023-03-27T10:16:14.990 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.186+0000 7f0c6910a700 1 --2- 172.21.15.159:0/2033899784 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0c64106160 0x7f0c64108540 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:14.990 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.186+0000 7f0c6910a700 1 -- 172.21.15.159:0/2033899784 >> 172.21.15.159:0/2033899784 conn(0x7f0c640ffba0 msgr2=0x7f0c64101fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:14.990 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.186+0000 7f0c6910a700 1 -- 172.21.15.159:0/2033899784 shutdown_connections 2023-03-27T10:16:14.990 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.186+0000 7f0c6910a700 1 -- 172.21.15.159:0/2033899784 wait complete. 2023-03-27T10:16:14.990 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.187+0000 7f0c6910a700 1 Processor -- start 2023-03-27T10:16:14.990 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.187+0000 7f0c6910a700 1 -- start start 2023-03-27T10:16:14.991 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.187+0000 7f0c6910a700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0c64106160 0x7f0c64199350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:14.991 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.187+0000 7f0c6910a700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c64199890 con 0x7f0c64106160 2023-03-27T10:16:14.991 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.187+0000 7f0c62d9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0c64106160 0x7f0c64199350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:14.991 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.187+0000 7f0c62d9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0c64106160 0x7f0c64199350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44524/0 (socket says 172.21.15.159:44524) 2023-03-27T10:16:14.991 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.187+0000 7f0c62d9d700 1 -- 172.21.15.159:0/98086894 learned_addr learned my addr 172.21.15.159:0/98086894 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:14.991 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.188+0000 7f0c62d9d700 1 -- 172.21.15.159:0/98086894 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0c54009550 con 0x7f0c64106160 2023-03-27T10:16:14.991 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.188+0000 7f0c62d9d700 1 --2- 172.21.15.159:0/98086894 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0c64106160 0x7f0c64199350 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f0c54009130 tx=0x7f0c54011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:14.991 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.188+0000 7f0c4bfff700 1 -- 172.21.15.159:0/98086894 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f0c54003ab0 con 0x7f0c64106160 2023-03-27T10:16:14.992 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.188+0000 7f0c4bfff700 1 -- 172.21.15.159:0/98086894 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f0c54003c10 con 0x7f0c64106160 2023-03-27T10:16:14.992 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.188+0000 7f0c6910a700 1 -- 172.21.15.159:0/98086894 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0c641098e0 con 0x7f0c64106160 2023-03-27T10:16:14.992 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.188+0000 7f0c4bfff700 1 -- 172.21.15.159:0/98086894 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f0c540247d0 con 0x7f0c64106160 2023-03-27T10:16:14.992 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.188+0000 7f0c6910a700 1 -- 172.21.15.159:0/98086894 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0c64109750 con 0x7f0c64106160 2023-03-27T10:16:14.992 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.189+0000 7f0c4bfff700 1 -- 172.21.15.159:0/98086894 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 8) v1 ==== 43509+0+0 (secure 0 0 0) 0x7f0c54017020 con 0x7f0c64106160 2023-03-27T10:16:14.992 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.189+0000 7f0c4bfff700 1 --2- 172.21.15.159:0/98086894 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f0c4c036870 0x7f0c4c038d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:14.992 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.189+0000 7f0c4bfff700 1 -- 172.21.15.159:0/98086894 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f0c54049d60 con 0x7f0c64106160 2023-03-27T10:16:14.992 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.191+0000 7f0c6259c700 1 --2- 172.21.15.159:0/98086894 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f0c4c036870 0x7f0c4c038d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:14.993 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.191+0000 7f0c6910a700 1 -- 172.21.15.159:0/98086894 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0c64109a70 con 0x7f0c64106160 2023-03-27T10:16:14.993 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.191+0000 7f0c6259c700 1 --2- 172.21.15.159:0/98086894 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f0c4c036870 0x7f0c4c038d20 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f0c58006fd0 tx=0x7f0c58006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:14.993 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.196+0000 7f0c4bfff700 1 -- 172.21.15.159:0/98086894 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f0c540193c0 con 0x7f0c64106160 2023-03-27T10:16:14.993 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.405+0000 7f0c6910a700 1 -- 172.21.15.159:0/98086894 --> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}) v1 -- 0x7f0c641010a0 con 0x7f0c4c036870 2023-03-27T10:16:14.993 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.412+0000 7f0c4bfff700 1 -- 172.21.15.159:0/98086894 <== mgr.14120 v2:172.21.15.159:6800/3561797228 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7f0c641010a0 con 0x7f0c4c036870 2023-03-27T10:16:14.993 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.414+0000 7f0c6910a700 1 -- 172.21.15.159:0/98086894 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f0c4c036870 msgr2=0x7f0c4c038d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:14.993 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.414+0000 7f0c6910a700 1 --2- 172.21.15.159:0/98086894 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f0c4c036870 0x7f0c4c038d20 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f0c58006fd0 tx=0x7f0c58006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:14.994 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.414+0000 7f0c6910a700 1 -- 172.21.15.159:0/98086894 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0c64106160 msgr2=0x7f0c64199350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:14.994 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.414+0000 7f0c6910a700 1 --2- 172.21.15.159:0/98086894 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0c64106160 0x7f0c64199350 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f0c54009130 tx=0x7f0c54011ad0 comp rx=0 tx=0).stop 2023-03-27T10:16:14.994 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.414+0000 7f0c6910a700 1 -- 172.21.15.159:0/98086894 shutdown_connections 2023-03-27T10:16:14.994 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.414+0000 7f0c6910a700 1 --2- 172.21.15.159:0/98086894 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f0c4c036870 0x7f0c4c038d20 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:14.994 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.414+0000 7f0c6910a700 1 --2- 172.21.15.159:0/98086894 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0c64106160 0x7f0c64199350 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:14.994 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.414+0000 7f0c6910a700 1 -- 172.21.15.159:0/98086894 >> 172.21.15.159:0/98086894 conn(0x7f0c640ffba0 msgr2=0x7f0c64100990 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:14.994 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.414+0000 7f0c6910a700 1 -- 172.21.15.159:0/98086894 shutdown_connections 2023-03-27T10:16:14.994 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:14.414+0000 7f0c6910a700 1 -- 172.21.15.159:0/98086894 wait complete. 2023-03-27T10:16:14.995 INFO:teuthology.orchestra.run.smithi159.stdout:Deploying unmanaged mgr service... 2023-03-27T10:16:15.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:15 smithi159 ceph-mon[93039]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:16:15.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:15 smithi159 ceph-mon[93039]: Saving service mon spec with placement count:5 2023-03-27T10:16:15.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:15 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' 2023-03-27T10:16:15.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:15 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' 2023-03-27T10:16:16.585 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-03-27T10:16:16.585 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.539+0000 7f99cefb7700 1 Processor -- start 2023-03-27T10:16:16.585 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.539+0000 7f99cefb7700 1 -- start start 2023-03-27T10:16:16.585 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.539+0000 7f99cefb7700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f99c8106cb0 0x7f99c81070c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:16.585 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.539+0000 7f99cefb7700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99c8107600 con 0x7f99c8106cb0 2023-03-27T10:16:16.586 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.539+0000 7f99ccd53700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f99c8106cb0 0x7f99c81070c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:16.586 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.539+0000 7f99ccd53700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f99c8106cb0 0x7f99c81070c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44528/0 (socket says 172.21.15.159:44528) 2023-03-27T10:16:16.586 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.539+0000 7f99ccd53700 1 -- 172.21.15.159:0/3028024336 learned_addr learned my addr 172.21.15.159:0/3028024336 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:16.586 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.539+0000 7f99ccd53700 1 -- 172.21.15.159:0/3028024336 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99c8107740 con 0x7f99c8106cb0 2023-03-27T10:16:16.586 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.540+0000 7f99ccd53700 1 --2- 172.21.15.159:0/3028024336 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f99c8106cb0 0x7f99c81070c0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f99bc0098a0 tx=0x7f99bc009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c63c371e1d9fa480 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:16.586 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.540+0000 7f99c77fe700 1 -- 172.21.15.159:0/3028024336 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f99bc004160 con 0x7f99c8106cb0 2023-03-27T10:16:16.586 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.540+0000 7f99c77fe700 1 -- 172.21.15.159:0/3028024336 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f99bc0042c0 con 0x7f99c8106cb0 2023-03-27T10:16:16.587 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.540+0000 7f99c77fe700 1 -- 172.21.15.159:0/3028024336 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f99bc0119b0 con 0x7f99c8106cb0 2023-03-27T10:16:16.587 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.540+0000 7f99cefb7700 1 -- 172.21.15.159:0/3028024336 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f99c8106cb0 msgr2=0x7f99c81070c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:16.587 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.540+0000 7f99cefb7700 1 --2- 172.21.15.159:0/3028024336 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f99c8106cb0 0x7f99c81070c0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f99bc0098a0 tx=0x7f99bc009bb0 comp rx=0 tx=0).stop 2023-03-27T10:16:16.587 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.540+0000 7f99cefb7700 1 -- 172.21.15.159:0/3028024336 shutdown_connections 2023-03-27T10:16:16.587 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.540+0000 7f99cefb7700 1 --2- 172.21.15.159:0/3028024336 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f99c8106cb0 0x7f99c81070c0 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:16.587 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.540+0000 7f99cefb7700 1 -- 172.21.15.159:0/3028024336 >> 172.21.15.159:0/3028024336 conn(0x7f99c80ff3c0 msgr2=0x7f99c81017d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:16.587 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.540+0000 7f99cefb7700 1 -- 172.21.15.159:0/3028024336 shutdown_connections 2023-03-27T10:16:16.587 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.540+0000 7f99cefb7700 1 -- 172.21.15.159:0/3028024336 wait complete. 2023-03-27T10:16:16.588 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.541+0000 7f99cefb7700 1 Processor -- start 2023-03-27T10:16:16.588 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.541+0000 7f99cefb7700 1 -- start start 2023-03-27T10:16:16.588 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.541+0000 7f99cefb7700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f99c8106cb0 0x7f99c81063b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:16.588 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.541+0000 7f99cefb7700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99c81068f0 con 0x7f99c8106cb0 2023-03-27T10:16:16.588 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.541+0000 7f99ccd53700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f99c8106cb0 0x7f99c81063b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:16.588 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.541+0000 7f99ccd53700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f99c8106cb0 0x7f99c81063b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44530/0 (socket says 172.21.15.159:44530) 2023-03-27T10:16:16.588 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.541+0000 7f99ccd53700 1 -- 172.21.15.159:0/1822777127 learned_addr learned my addr 172.21.15.159:0/1822777127 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:16.588 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.541+0000 7f99ccd53700 1 -- 172.21.15.159:0/1822777127 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99bc009550 con 0x7f99c8106cb0 2023-03-27T10:16:16.589 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.542+0000 7f99ccd53700 1 --2- 172.21.15.159:0/1822777127 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f99c8106cb0 0x7f99c81063b0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f99bc000c00 tx=0x7f99bc011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:16.589 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.542+0000 7f99c5ffb700 1 -- 172.21.15.159:0/1822777127 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f99bc0039f0 con 0x7f99c8106cb0 2023-03-27T10:16:16.589 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.542+0000 7f99c5ffb700 1 -- 172.21.15.159:0/1822777127 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f99bc003b50 con 0x7f99c8106cb0 2023-03-27T10:16:16.589 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.542+0000 7f99c5ffb700 1 -- 172.21.15.159:0/1822777127 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f99bc01a780 con 0x7f99c8106cb0 2023-03-27T10:16:16.589 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.542+0000 7f99cefb7700 1 -- 172.21.15.159:0/1822777127 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f99c81044b0 con 0x7f99c8106cb0 2023-03-27T10:16:16.589 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.542+0000 7f99cefb7700 1 -- 172.21.15.159:0/1822777127 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f99c8104320 con 0x7f99c8106cb0 2023-03-27T10:16:16.589 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.543+0000 7f99c5ffb700 1 -- 172.21.15.159:0/1822777127 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 8) v1 ==== 43509+0+0 (secure 0 0 0) 0x7f99bc017020 con 0x7f99c8106cb0 2023-03-27T10:16:16.590 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.543+0000 7f99cefb7700 1 -- 172.21.15.159:0/1822777127 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f99c8061e00 con 0x7f99c8106cb0 2023-03-27T10:16:16.590 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.543+0000 7f99c5ffb700 1 --2- 172.21.15.159:0/1822777127 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f99b8036870 0x7f99b8038d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:16.590 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.543+0000 7f99c5ffb700 1 -- 172.21.15.159:0/1822777127 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f99bc049b60 con 0x7f99c8106cb0 2023-03-27T10:16:16.590 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.543+0000 7f99c7fff700 1 --2- 172.21.15.159:0/1822777127 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f99b8036870 0x7f99b8038d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:16.590 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.543+0000 7f99c7fff700 1 --2- 172.21.15.159:0/1822777127 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f99b8036870 0x7f99b8038d20 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f99b4006fd0 tx=0x7f99b4006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:16.590 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.548+0000 7f99c5ffb700 1 -- 172.21.15.159:0/1822777127 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f99bc02d3f0 con 0x7f99c8106cb0 2023-03-27T10:16:16.591 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.759+0000 7f99cefb7700 1 -- 172.21.15.159:0/1822777127 --> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}) v1 -- 0x7f99c8105900 con 0x7f99b8036870 2023-03-27T10:16:16.591 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.764+0000 7f99c5ffb700 1 -- 172.21.15.159:0/1822777127 <== mgr.14120 v2:172.21.15.159:6800/3561797228 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7f99c8105900 con 0x7f99b8036870 2023-03-27T10:16:16.591 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.766+0000 7f99cefb7700 1 -- 172.21.15.159:0/1822777127 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f99b8036870 msgr2=0x7f99b8038d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:16.591 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.766+0000 7f99cefb7700 1 --2- 172.21.15.159:0/1822777127 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f99b8036870 0x7f99b8038d20 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f99b4006fd0 tx=0x7f99b4006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:16.591 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.766+0000 7f99cefb7700 1 -- 172.21.15.159:0/1822777127 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f99c8106cb0 msgr2=0x7f99c81063b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:16.591 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.766+0000 7f99cefb7700 1 --2- 172.21.15.159:0/1822777127 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f99c8106cb0 0x7f99c81063b0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f99bc000c00 tx=0x7f99bc011a30 comp rx=0 tx=0).stop 2023-03-27T10:16:16.591 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.766+0000 7f99cefb7700 1 -- 172.21.15.159:0/1822777127 shutdown_connections 2023-03-27T10:16:16.591 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.766+0000 7f99cefb7700 1 --2- 172.21.15.159:0/1822777127 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f99b8036870 0x7f99b8038d20 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:16.592 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.766+0000 7f99cefb7700 1 --2- 172.21.15.159:0/1822777127 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f99c8106cb0 0x7f99c81063b0 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:16.592 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.766+0000 7f99cefb7700 1 -- 172.21.15.159:0/1822777127 >> 172.21.15.159:0/1822777127 conn(0x7f99c80ff3c0 msgr2=0x7f99c81000b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:16.592 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.766+0000 7f99cefb7700 1 -- 172.21.15.159:0/1822777127 shutdown_connections 2023-03-27T10:16:16.592 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:15.766+0000 7f99cefb7700 1 -- 172.21.15.159:0/1822777127 wait complete. 2023-03-27T10:16:17.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:16 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' 2023-03-27T10:16:17.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:16 smithi159 ceph-mon[93039]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:16:17.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:16 smithi159 ceph-mon[93039]: Saving service mgr spec with placement count:2 2023-03-27T10:16:17.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:16 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' 2023-03-27T10:16:17.630 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.085+0000 7f78b8864700 1 Processor -- start 2023-03-27T10:16:17.630 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.085+0000 7f78b8864700 1 -- start start 2023-03-27T10:16:17.630 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.085+0000 7f78b8864700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f78b4104540 0x7f78b4104950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:17.631 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.085+0000 7f78b8864700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78b4104e90 con 0x7f78b4104540 2023-03-27T10:16:17.631 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.086+0000 7f78b259c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f78b4104540 0x7f78b4104950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:17.631 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.086+0000 7f78b259c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f78b4104540 0x7f78b4104950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44534/0 (socket says 172.21.15.159:44534) 2023-03-27T10:16:17.631 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.086+0000 7f78b259c700 1 -- 172.21.15.159:0/1078357990 learned_addr learned my addr 172.21.15.159:0/1078357990 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:17.631 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.086+0000 7f78b259c700 1 -- 172.21.15.159:0/1078357990 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f78b4104fd0 con 0x7f78b4104540 2023-03-27T10:16:17.631 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.086+0000 7f78b259c700 1 --2- 172.21.15.159:0/1078357990 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f78b4104540 0x7f78b4104950 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f78a40098a0 tx=0x7f78a4009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9f0700110d11683a server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:17.631 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.086+0000 7f78b159a700 1 -- 172.21.15.159:0/1078357990 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f78a4004160 con 0x7f78b4104540 2023-03-27T10:16:17.631 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.086+0000 7f78b159a700 1 -- 172.21.15.159:0/1078357990 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f78a40042c0 con 0x7f78b4104540 2023-03-27T10:16:17.632 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.086+0000 7f78b159a700 1 -- 172.21.15.159:0/1078357990 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f78a40119b0 con 0x7f78b4104540 2023-03-27T10:16:17.632 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.087+0000 7f78b8864700 1 -- 172.21.15.159:0/1078357990 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f78b4104540 msgr2=0x7f78b4104950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:17.632 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.087+0000 7f78b8864700 1 --2- 172.21.15.159:0/1078357990 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f78b4104540 0x7f78b4104950 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f78a40098a0 tx=0x7f78a4009bb0 comp rx=0 tx=0).stop 2023-03-27T10:16:17.633 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.087+0000 7f78b8864700 1 -- 172.21.15.159:0/1078357990 shutdown_connections 2023-03-27T10:16:17.633 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.087+0000 7f78b8864700 1 --2- 172.21.15.159:0/1078357990 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f78b4104540 0x7f78b4104950 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:17.633 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.087+0000 7f78b8864700 1 -- 172.21.15.159:0/1078357990 >> 172.21.15.159:0/1078357990 conn(0x7f78b40ffb90 msgr2=0x7f78b4101fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:17.633 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.087+0000 7f78b8864700 1 -- 172.21.15.159:0/1078357990 shutdown_connections 2023-03-27T10:16:17.633 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.087+0000 7f78b8864700 1 -- 172.21.15.159:0/1078357990 wait complete. 2023-03-27T10:16:17.634 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.087+0000 7f78b8864700 1 Processor -- start 2023-03-27T10:16:17.634 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.088+0000 7f78b8864700 1 -- start start 2023-03-27T10:16:17.634 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.088+0000 7f78b8864700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f78b4104540 0x7f78b4199340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:17.634 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.088+0000 7f78b8864700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78b4199880 con 0x7f78b4104540 2023-03-27T10:16:17.634 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.088+0000 7f78b259c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f78b4104540 0x7f78b4199340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:17.634 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.088+0000 7f78b259c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f78b4104540 0x7f78b4199340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44536/0 (socket says 172.21.15.159:44536) 2023-03-27T10:16:17.634 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.088+0000 7f78b259c700 1 -- 172.21.15.159:0/1974579427 learned_addr learned my addr 172.21.15.159:0/1974579427 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:17.635 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.088+0000 7f78b259c700 1 -- 172.21.15.159:0/1974579427 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f78a4009550 con 0x7f78b4104540 2023-03-27T10:16:17.635 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.088+0000 7f78b259c700 1 --2- 172.21.15.159:0/1974579427 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f78b4104540 0x7f78b4199340 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f78a4003f00 tx=0x7f78a4011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:17.635 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.088+0000 7f78a37fe700 1 -- 172.21.15.159:0/1974579427 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f78a4003b20 con 0x7f78b4104540 2023-03-27T10:16:17.635 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.088+0000 7f78a37fe700 1 -- 172.21.15.159:0/1974579427 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f78a4003c80 con 0x7f78b4104540 2023-03-27T10:16:17.635 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.088+0000 7f78b8864700 1 -- 172.21.15.159:0/1974579427 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f78b41065d0 con 0x7f78b4104540 2023-03-27T10:16:17.635 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.088+0000 7f78a37fe700 1 -- 172.21.15.159:0/1974579427 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f78a4019770 con 0x7f78b4104540 2023-03-27T10:16:17.635 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.088+0000 7f78b8864700 1 -- 172.21.15.159:0/1974579427 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f78b4106440 con 0x7f78b4104540 2023-03-27T10:16:17.635 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.089+0000 7f78a37fe700 1 -- 172.21.15.159:0/1974579427 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 8) v1 ==== 43509+0+0 (secure 0 0 0) 0x7f78a402c070 con 0x7f78b4104540 2023-03-27T10:16:17.636 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.089+0000 7f78a37fe700 1 --2- 172.21.15.159:0/1974579427 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f789c036870 0x7f789c038d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:17.636 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.089+0000 7f78b8864700 1 -- 172.21.15.159:0/1974579427 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f78b4108a10 con 0x7f78b4104540 2023-03-27T10:16:17.636 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.089+0000 7f78a37fe700 1 -- 172.21.15.159:0/1974579427 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f78a4049dc0 con 0x7f78b4104540 2023-03-27T10:16:17.636 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.089+0000 7f78b1d9b700 1 --2- 172.21.15.159:0/1974579427 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f789c036870 0x7f789c038d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:17.636 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.090+0000 7f78b1d9b700 1 --2- 172.21.15.159:0/1974579427 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f789c036870 0x7f789c038d20 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f78a8006fd0 tx=0x7f78a8006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:17.636 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.094+0000 7f78a37fe700 1 -- 172.21.15.159:0/1974579427 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f78a400b3e0 con 0x7f78b4104540 2023-03-27T10:16:17.636 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.283+0000 7f78b8864700 1 -- 172.21.15.159:0/1974579427 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/container_init}] v 0) v1 -- 0x7f78b419cec0 con 0x7f78b4104540 2023-03-27T10:16:17.637 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.286+0000 7f78a37fe700 1 -- 172.21.15.159:0/1974579427 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/container_init}]=0 v7)=0 v7) v1 ==== 142+0+0 (secure 0 0 0) 0x7f78a40172b0 con 0x7f78b4104540 2023-03-27T10:16:17.637 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.288+0000 7f78b8864700 1 -- 172.21.15.159:0/1974579427 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f789c036870 msgr2=0x7f789c038d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:17.637 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.288+0000 7f78b8864700 1 --2- 172.21.15.159:0/1974579427 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f789c036870 0x7f789c038d20 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f78a8006fd0 tx=0x7f78a8006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:17.637 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.288+0000 7f78b8864700 1 -- 172.21.15.159:0/1974579427 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f78b4104540 msgr2=0x7f78b4199340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:17.637 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.288+0000 7f78b8864700 1 --2- 172.21.15.159:0/1974579427 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f78b4104540 0x7f78b4199340 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f78a4003f00 tx=0x7f78a4011a30 comp rx=0 tx=0).stop 2023-03-27T10:16:17.637 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.288+0000 7f78b8864700 1 -- 172.21.15.159:0/1974579427 shutdown_connections 2023-03-27T10:16:17.637 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.288+0000 7f78b8864700 1 --2- 172.21.15.159:0/1974579427 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f789c036870 0x7f789c038d20 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:17.637 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.288+0000 7f78b8864700 1 --2- 172.21.15.159:0/1974579427 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f78b4104540 0x7f78b4199340 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:17.638 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.288+0000 7f78b8864700 1 -- 172.21.15.159:0/1974579427 >> 172.21.15.159:0/1974579427 conn(0x7f78b40ffb90 msgr2=0x7f78b40737c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:17.638 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.289+0000 7f78b8864700 1 -- 172.21.15.159:0/1974579427 shutdown_connections 2023-03-27T10:16:17.638 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:17.289+0000 7f78b8864700 1 -- 172.21.15.159:0/1974579427 wait complete. 2023-03-27T10:16:18.572 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:18 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/1974579427' entity='client.admin' 2023-03-27T10:16:18.825 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.233+0000 7f81ae708700 1 Processor -- start 2023-03-27T10:16:18.825 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.233+0000 7f81ae708700 1 -- start start 2023-03-27T10:16:18.825 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.233+0000 7f81ae708700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f81a8104540 0x7f81a8104950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:18.825 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.233+0000 7f81ae708700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81a8104e90 con 0x7f81a8104540 2023-03-27T10:16:18.825 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.233+0000 7f81a7fff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f81a8104540 0x7f81a8104950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:18.826 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.234+0000 7f81a7fff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f81a8104540 0x7f81a8104950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44540/0 (socket says 172.21.15.159:44540) 2023-03-27T10:16:18.826 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.234+0000 7f81a7fff700 1 -- 172.21.15.159:0/2321341064 learned_addr learned my addr 172.21.15.159:0/2321341064 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:18.826 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.234+0000 7f81a7fff700 1 -- 172.21.15.159:0/2321341064 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81a8104fd0 con 0x7f81a8104540 2023-03-27T10:16:18.826 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.234+0000 7f81a7fff700 1 --2- 172.21.15.159:0/2321341064 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f81a8104540 0x7f81a8104950 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f81980098a0 tx=0x7f8198009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8ed8ce75bc542c88 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:18.826 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.234+0000 7f81a6ffd700 1 -- 172.21.15.159:0/2321341064 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f8198004160 con 0x7f81a8104540 2023-03-27T10:16:18.826 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.234+0000 7f81a6ffd700 1 -- 172.21.15.159:0/2321341064 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f81980042c0 con 0x7f81a8104540 2023-03-27T10:16:18.826 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.234+0000 7f81a6ffd700 1 -- 172.21.15.159:0/2321341064 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f81980119b0 con 0x7f81a8104540 2023-03-27T10:16:18.826 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.235+0000 7f81ae708700 1 -- 172.21.15.159:0/2321341064 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f81a8104540 msgr2=0x7f81a8104950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:18.827 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.235+0000 7f81ae708700 1 --2- 172.21.15.159:0/2321341064 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f81a8104540 0x7f81a8104950 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f81980098a0 tx=0x7f8198009bb0 comp rx=0 tx=0).stop 2023-03-27T10:16:18.827 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.235+0000 7f81ae708700 1 -- 172.21.15.159:0/2321341064 shutdown_connections 2023-03-27T10:16:18.827 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.235+0000 7f81ae708700 1 --2- 172.21.15.159:0/2321341064 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f81a8104540 0x7f81a8104950 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:18.827 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.235+0000 7f81ae708700 1 -- 172.21.15.159:0/2321341064 >> 172.21.15.159:0/2321341064 conn(0x7f81a80ffb90 msgr2=0x7f81a8101fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:18.827 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.235+0000 7f81ae708700 1 -- 172.21.15.159:0/2321341064 shutdown_connections 2023-03-27T10:16:18.827 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.235+0000 7f81ae708700 1 -- 172.21.15.159:0/2321341064 wait complete. 2023-03-27T10:16:18.827 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.235+0000 7f81ae708700 1 Processor -- start 2023-03-27T10:16:18.828 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.235+0000 7f81ae708700 1 -- start start 2023-03-27T10:16:18.828 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.236+0000 7f81ae708700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f81a8104540 0x7f81a80739f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:18.828 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.236+0000 7f81ae708700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81a819c070 con 0x7f81a8104540 2023-03-27T10:16:18.828 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.236+0000 7f81a7fff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f81a8104540 0x7f81a80739f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:18.828 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.236+0000 7f81a7fff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f81a8104540 0x7f81a80739f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44542/0 (socket says 172.21.15.159:44542) 2023-03-27T10:16:18.828 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.236+0000 7f81a7fff700 1 -- 172.21.15.159:0/2122488481 learned_addr learned my addr 172.21.15.159:0/2122488481 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:18.829 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.236+0000 7f81a7fff700 1 -- 172.21.15.159:0/2122488481 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8198009550 con 0x7f81a8104540 2023-03-27T10:16:18.829 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.236+0000 7f81a7fff700 1 --2- 172.21.15.159:0/2122488481 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f81a8104540 0x7f81a80739f0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f8198006af0 tx=0x7f8198011a10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:18.829 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.236+0000 7f81a57fa700 1 -- 172.21.15.159:0/2122488481 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f8198003a40 con 0x7f81a8104540 2023-03-27T10:16:18.829 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.236+0000 7f81a57fa700 1 -- 172.21.15.159:0/2122488481 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8198003ba0 con 0x7f81a8104540 2023-03-27T10:16:18.829 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.236+0000 7f81ae708700 1 -- 172.21.15.159:0/2122488481 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f81a81065d0 con 0x7f81a8104540 2023-03-27T10:16:18.829 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.236+0000 7f81a57fa700 1 -- 172.21.15.159:0/2122488481 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f81980196c0 con 0x7f81a8104540 2023-03-27T10:16:18.829 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.236+0000 7f81ae708700 1 -- 172.21.15.159:0/2122488481 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f81a8106440 con 0x7f81a8104540 2023-03-27T10:16:18.830 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.237+0000 7f81a57fa700 1 -- 172.21.15.159:0/2122488481 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 8) v1 ==== 43509+0+0 (secure 0 0 0) 0x7f819802a020 con 0x7f81a8104540 2023-03-27T10:16:18.830 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.237+0000 7f81a57fa700 1 --2- 172.21.15.159:0/2122488481 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f81940367d0 0x7f8194038c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:18.830 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.237+0000 7f81a57fa700 1 -- 172.21.15.159:0/2122488481 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f8198049cc0 con 0x7f81a8104540 2023-03-27T10:16:18.830 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.237+0000 7f81ae708700 1 -- 172.21.15.159:0/2122488481 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f81a8061e00 con 0x7f81a8104540 2023-03-27T10:16:18.830 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.237+0000 7f81a77fe700 1 --2- 172.21.15.159:0/2122488481 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f81940367d0 0x7f8194038c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:18.830 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.238+0000 7f81a77fe700 1 --2- 172.21.15.159:0/2122488481 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f81940367d0 0x7f8194038c80 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f8190006fd0 tx=0x7f8190006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:18.830 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.242+0000 7f81a57fa700 1 -- 172.21.15.159:0/2122488481 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f8198019980 con 0x7f81a8104540 2023-03-27T10:16:18.831 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.440+0000 7f81ae708700 1 -- 172.21.15.159:0/2122488481 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command([{prefix=config set, name=mgr/dashboard/ssl_server_port}] v 0) v1 -- 0x7f81a819ca00 con 0x7f81a8104540 2023-03-27T10:16:18.831 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.443+0000 7f81a57fa700 1 -- 172.21.15.159:0/2122488481 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/dashboard/ssl_server_port}]=0 v8)=0 v8) v1 ==== 130+0+0 (secure 0 0 0) 0x7f8198023390 con 0x7f81a8104540 2023-03-27T10:16:18.831 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.445+0000 7f81ae708700 1 -- 172.21.15.159:0/2122488481 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f81940367d0 msgr2=0x7f8194038c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:18.831 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.445+0000 7f81ae708700 1 --2- 172.21.15.159:0/2122488481 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f81940367d0 0x7f8194038c80 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f8190006fd0 tx=0x7f8190006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:18.831 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.445+0000 7f81ae708700 1 -- 172.21.15.159:0/2122488481 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f81a8104540 msgr2=0x7f81a80739f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:18.831 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.445+0000 7f81ae708700 1 --2- 172.21.15.159:0/2122488481 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f81a8104540 0x7f81a80739f0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f8198006af0 tx=0x7f8198011a10 comp rx=0 tx=0).stop 2023-03-27T10:16:18.831 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.445+0000 7f81ae708700 1 -- 172.21.15.159:0/2122488481 shutdown_connections 2023-03-27T10:16:18.831 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.445+0000 7f81ae708700 1 --2- 172.21.15.159:0/2122488481 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f81940367d0 0x7f8194038c80 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:18.832 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.445+0000 7f81ae708700 1 --2- 172.21.15.159:0/2122488481 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f81a8104540 0x7f81a80739f0 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:18.832 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.446+0000 7f81ae708700 1 -- 172.21.15.159:0/2122488481 >> 172.21.15.159:0/2122488481 conn(0x7f81a80ffb90 msgr2=0x7f81a81008e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:18.832 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.446+0000 7f81ae708700 1 -- 172.21.15.159:0/2122488481 shutdown_connections 2023-03-27T10:16:18.832 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:18.446+0000 7f81ae708700 1 -- 172.21.15.159:0/2122488481 wait complete. 2023-03-27T10:16:18.832 INFO:teuthology.orchestra.run.smithi159.stdout:Enabling the dashboard module... 2023-03-27T10:16:19.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:19 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' 2023-03-27T10:16:19.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:19 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/2122488481' entity='client.admin' 2023-03-27T10:16:20.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:20 smithi159 ceph-mon[93039]: from='mgr.14120 172.21.15.159:0/4285792993' entity='mgr.x' 2023-03-27T10:16:20.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:20 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/1093527084' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-03-27T10:16:20.616 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:20 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: ignoring --setuser ceph since I am not root 2023-03-27T10:16:20.616 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:20 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: ignoring --setgroup ceph since I am not root 2023-03-27T10:16:20.872 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.349+0000 7effad8f3700 1 Processor -- start 2023-03-27T10:16:20.872 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.349+0000 7effad8f3700 1 -- start start 2023-03-27T10:16:20.873 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.350+0000 7effad8f3700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7effa81061a0 0x7effa8108580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:20.873 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.350+0000 7effad8f3700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7effa8073c00 con 0x7effa81061a0 2023-03-27T10:16:20.873 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.350+0000 7effa6ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7effa81061a0 0x7effa8108580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:20.873 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.350+0000 7effa6ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7effa81061a0 0x7effa8108580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44546/0 (socket says 172.21.15.159:44546) 2023-03-27T10:16:20.873 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.350+0000 7effa6ffd700 1 -- 172.21.15.159:0/977942896 learned_addr learned my addr 172.21.15.159:0/977942896 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:20.873 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.350+0000 7effa6ffd700 1 -- 172.21.15.159:0/977942896 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7effa8073d40 con 0x7effa81061a0 2023-03-27T10:16:20.873 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.350+0000 7effa6ffd700 1 --2- 172.21.15.159:0/977942896 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7effa81061a0 0x7effa8108580 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7eff900098a0 tx=0x7eff90009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e51c4696b24f2826 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:20.874 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.350+0000 7effa5ffb700 1 -- 172.21.15.159:0/977942896 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7eff90004160 con 0x7effa81061a0 2023-03-27T10:16:20.874 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.351+0000 7effa5ffb700 1 -- 172.21.15.159:0/977942896 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7eff900042c0 con 0x7effa81061a0 2023-03-27T10:16:20.874 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.351+0000 7effa5ffb700 1 -- 172.21.15.159:0/977942896 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7eff900119b0 con 0x7effa81061a0 2023-03-27T10:16:20.874 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.351+0000 7effad8f3700 1 -- 172.21.15.159:0/977942896 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7effa81061a0 msgr2=0x7effa8108580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:20.874 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.351+0000 7effad8f3700 1 --2- 172.21.15.159:0/977942896 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7effa81061a0 0x7effa8108580 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7eff900098a0 tx=0x7eff90009bb0 comp rx=0 tx=0).stop 2023-03-27T10:16:20.874 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.351+0000 7effad8f3700 1 -- 172.21.15.159:0/977942896 shutdown_connections 2023-03-27T10:16:20.874 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.351+0000 7effad8f3700 1 --2- 172.21.15.159:0/977942896 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7effa81061a0 0x7effa8108580 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:20.875 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.351+0000 7effad8f3700 1 -- 172.21.15.159:0/977942896 >> 172.21.15.159:0/977942896 conn(0x7effa80ffba0 msgr2=0x7effa8101fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:20.875 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.351+0000 7effad8f3700 1 -- 172.21.15.159:0/977942896 shutdown_connections 2023-03-27T10:16:20.875 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.351+0000 7effad8f3700 1 -- 172.21.15.159:0/977942896 wait complete. 2023-03-27T10:16:20.875 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.352+0000 7effad8f3700 1 Processor -- start 2023-03-27T10:16:20.875 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.352+0000 7effad8f3700 1 -- start start 2023-03-27T10:16:20.875 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.352+0000 7effad8f3700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7effa81061a0 0x7effa8199310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:20.875 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.352+0000 7effad8f3700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7effa8199850 con 0x7effa81061a0 2023-03-27T10:16:20.875 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.352+0000 7effa6ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7effa81061a0 0x7effa8199310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:20.876 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.352+0000 7effa6ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7effa81061a0 0x7effa8199310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44548/0 (socket says 172.21.15.159:44548) 2023-03-27T10:16:20.876 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.352+0000 7effa6ffd700 1 -- 172.21.15.159:0/1093527084 learned_addr learned my addr 172.21.15.159:0/1093527084 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:20.876 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.352+0000 7effa6ffd700 1 -- 172.21.15.159:0/1093527084 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff90009550 con 0x7effa81061a0 2023-03-27T10:16:20.876 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.353+0000 7effa6ffd700 1 --2- 172.21.15.159:0/1093527084 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7effa81061a0 0x7effa8199310 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7eff90006af0 tx=0x7eff90011a10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:20.876 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.353+0000 7effac8f1700 1 -- 172.21.15.159:0/1093527084 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7eff90003a40 con 0x7effa81061a0 2023-03-27T10:16:20.876 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.353+0000 7effac8f1700 1 -- 172.21.15.159:0/1093527084 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7eff90003ba0 con 0x7effa81061a0 2023-03-27T10:16:20.876 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.353+0000 7effad8f3700 1 -- 172.21.15.159:0/1093527084 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7effa81098a0 con 0x7effa81061a0 2023-03-27T10:16:20.876 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.353+0000 7effac8f1700 1 -- 172.21.15.159:0/1093527084 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7eff90019690 con 0x7effa81061a0 2023-03-27T10:16:20.877 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.353+0000 7effad8f3700 1 -- 172.21.15.159:0/1093527084 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7effa8109710 con 0x7effa81061a0 2023-03-27T10:16:20.877 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.354+0000 7effac8f1700 1 -- 172.21.15.159:0/1093527084 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 8) v1 ==== 43509+0+0 (secure 0 0 0) 0x7eff9002a020 con 0x7effa81061a0 2023-03-27T10:16:20.877 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.354+0000 7effac8f1700 1 --2- 172.21.15.159:0/1093527084 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7eff94036820 0x7eff94038cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:20.877 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.354+0000 7effac8f1700 1 -- 172.21.15.159:0/1093527084 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7eff90049c70 con 0x7effa81061a0 2023-03-27T10:16:20.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.354+0000 7effad8f3700 1 -- 172.21.15.159:0/1093527084 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7effa8109a30 con 0x7effa81061a0 2023-03-27T10:16:20.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.354+0000 7effa67fc700 1 --2- 172.21.15.159:0/1093527084 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7eff94036820 0x7eff94038cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:20.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.354+0000 7effa67fc700 1 --2- 172.21.15.159:0/1093527084 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7eff94036820 0x7eff94038cd0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7eff98006fd0 tx=0x7eff98006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:20.878 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.358+0000 7effac8f1700 1 -- 172.21.15.159:0/1093527084 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7eff9002d430 con 0x7effa81061a0 2023-03-27T10:16:20.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:19.603+0000 7effad8f3700 1 -- 172.21.15.159:0/1093527084 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "dashboard"} v 0) v1 -- 0x7effa8109a30 con 0x7effa81061a0 2023-03-27T10:16:20.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:20.480+0000 7effac8f1700 1 -- 172.21.15.159:0/1093527084 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mgrmap(e 9) v1 ==== 43522+0+0 (secure 0 0 0) 0x7eff90033030 con 0x7effa81061a0 2023-03-27T10:16:20.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:20.481+0000 7effac8f1700 1 -- 172.21.15.159:0/1093527084 <== mon.0 v2:172.21.15.159:3300/0 8 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "dashboard"}]=0 v9) v1 ==== 88+0+0 (secure 0 0 0) 0x7eff9002add0 con 0x7effa81061a0 2023-03-27T10:16:20.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:20.483+0000 7effad8f3700 1 -- 172.21.15.159:0/1093527084 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7eff94036820 msgr2=0x7eff94038cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:20.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:20.483+0000 7effad8f3700 1 --2- 172.21.15.159:0/1093527084 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7eff94036820 0x7eff94038cd0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7eff98006fd0 tx=0x7eff98006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:20.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:20.483+0000 7effad8f3700 1 -- 172.21.15.159:0/1093527084 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7effa81061a0 msgr2=0x7effa8199310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:20.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:20.483+0000 7effad8f3700 1 --2- 172.21.15.159:0/1093527084 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7effa81061a0 0x7effa8199310 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7eff90006af0 tx=0x7eff90011a10 comp rx=0 tx=0).stop 2023-03-27T10:16:20.879 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:20.483+0000 7effad8f3700 1 -- 172.21.15.159:0/1093527084 shutdown_connections 2023-03-27T10:16:20.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:20.483+0000 7effad8f3700 1 --2- 172.21.15.159:0/1093527084 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7eff94036820 0x7eff94038cd0 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:20.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:20.483+0000 7effad8f3700 1 --2- 172.21.15.159:0/1093527084 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7effa81061a0 0x7effa8199310 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:20.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:20.483+0000 7effad8f3700 1 -- 172.21.15.159:0/1093527084 >> 172.21.15.159:0/1093527084 conn(0x7effa80ffba0 msgr2=0x7effa81009c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:20.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:20.483+0000 7effad8f3700 1 -- 172.21.15.159:0/1093527084 shutdown_connections 2023-03-27T10:16:20.880 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:20.483+0000 7effad8f3700 1 -- 172.21.15.159:0/1093527084 wait complete. 2023-03-27T10:16:20.909 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:20 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:20.659+0000 7fc9f3903200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-03-27T10:16:20.909 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:20 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:20.739+0000 7fc9f3903200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-03-27T10:16:20.909 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:20 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:20.830+0000 7fc9f3903200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-03-27T10:16:21.243 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:20 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:20.931+0000 7fc9f3903200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-03-27T10:16:21.243 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:21 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:21.104+0000 7fc9f3903200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-03-27T10:16:21.516 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:21 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/1093527084' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-03-27T10:16:21.516 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:21 smithi159 ceph-mon[93039]: mgrmap e9: x(active, since 15s) 2023-03-27T10:16:21.517 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:21 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:21.423+0000 7fc9f3903200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-03-27T10:16:21.848 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:21 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:21.516+0000 7fc9f3903200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-03-27T10:16:21.848 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:21 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:21.652+0000 7fc9f3903200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-03-27T10:16:21.848 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:21 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:21.747+0000 7fc9f3903200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-03-27T10:16:22.220 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout { 2023-03-27T10:16:22.220 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-03-27T10:16:22.221 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "available": true, 2023-03-27T10:16:22.221 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "active_name": "x", 2023-03-27T10:16:22.221 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-03-27T10:16:22.221 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout } 2023-03-27T10:16:22.221 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.570+0000 7ff333155700 1 Processor -- start 2023-03-27T10:16:22.221 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.570+0000 7ff333155700 1 -- start start 2023-03-27T10:16:22.222 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.570+0000 7ff333155700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff32c073bc0 0x7ff32c10a5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:22.222 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.570+0000 7ff333155700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff32c105fe0 con 0x7ff32c073bc0 2023-03-27T10:16:22.222 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.570+0000 7ff330ef1700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff32c073bc0 0x7ff32c10a5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:22.222 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.570+0000 7ff330ef1700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff32c073bc0 0x7ff32c10a5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44562/0 (socket says 172.21.15.159:44562) 2023-03-27T10:16:22.222 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.570+0000 7ff330ef1700 1 -- 172.21.15.159:0/3518520568 learned_addr learned my addr 172.21.15.159:0/3518520568 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:22.222 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.571+0000 7ff330ef1700 1 -- 172.21.15.159:0/3518520568 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff32c10ab30 con 0x7ff32c073bc0 2023-03-27T10:16:22.222 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.571+0000 7ff330ef1700 1 --2- 172.21.15.159:0/3518520568 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff32c073bc0 0x7ff32c10a5f0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7ff3200098a0 tx=0x7ff320009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c93524acdb243e3d server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:22.222 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.571+0000 7ff32b7fe700 1 -- 172.21.15.159:0/3518520568 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7ff320004160 con 0x7ff32c073bc0 2023-03-27T10:16:22.223 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.571+0000 7ff32b7fe700 1 -- 172.21.15.159:0/3518520568 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff3200042c0 con 0x7ff32c073bc0 2023-03-27T10:16:22.223 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.571+0000 7ff32b7fe700 1 -- 172.21.15.159:0/3518520568 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7ff3200119b0 con 0x7ff32c073bc0 2023-03-27T10:16:22.223 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.572+0000 7ff333155700 1 -- 172.21.15.159:0/3518520568 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff32c073bc0 msgr2=0x7ff32c10a5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:22.223 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.572+0000 7ff333155700 1 --2- 172.21.15.159:0/3518520568 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff32c073bc0 0x7ff32c10a5f0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7ff3200098a0 tx=0x7ff320009bb0 comp rx=0 tx=0).stop 2023-03-27T10:16:22.223 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.572+0000 7ff333155700 1 -- 172.21.15.159:0/3518520568 shutdown_connections 2023-03-27T10:16:22.223 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.572+0000 7ff333155700 1 --2- 172.21.15.159:0/3518520568 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff32c073bc0 0x7ff32c10a5f0 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:22.223 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.572+0000 7ff333155700 1 -- 172.21.15.159:0/3518520568 >> 172.21.15.159:0/3518520568 conn(0x7ff32c0ffba0 msgr2=0x7ff32c101fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:22.223 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.572+0000 7ff333155700 1 -- 172.21.15.159:0/3518520568 shutdown_connections 2023-03-27T10:16:22.224 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.572+0000 7ff333155700 1 -- 172.21.15.159:0/3518520568 wait complete. 2023-03-27T10:16:22.224 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.572+0000 7ff333155700 1 Processor -- start 2023-03-27T10:16:22.224 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.572+0000 7ff333155700 1 -- start start 2023-03-27T10:16:22.224 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.573+0000 7ff333155700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff32c19b570 0x7ff32c19b980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:22.224 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.573+0000 7ff333155700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff32c19bec0 con 0x7ff32c19b570 2023-03-27T10:16:22.224 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.573+0000 7ff330ef1700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff32c19b570 0x7ff32c19b980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:22.224 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.573+0000 7ff330ef1700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff32c19b570 0x7ff32c19b980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44564/0 (socket says 172.21.15.159:44564) 2023-03-27T10:16:22.225 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.573+0000 7ff330ef1700 1 -- 172.21.15.159:0/995539323 learned_addr learned my addr 172.21.15.159:0/995539323 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:22.225 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.573+0000 7ff330ef1700 1 -- 172.21.15.159:0/995539323 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff320009550 con 0x7ff32c19b570 2023-03-27T10:16:22.225 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.573+0000 7ff330ef1700 1 --2- 172.21.15.159:0/995539323 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff32c19b570 0x7ff32c19b980 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7ff320003e60 tx=0x7ff320011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:22.225 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.573+0000 7ff329ffb700 1 -- 172.21.15.159:0/995539323 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7ff320003a60 con 0x7ff32c19b570 2023-03-27T10:16:22.225 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.573+0000 7ff329ffb700 1 -- 172.21.15.159:0/995539323 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff320003bc0 con 0x7ff32c19b570 2023-03-27T10:16:22.225 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.573+0000 7ff333155700 1 -- 172.21.15.159:0/995539323 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff32c10bb00 con 0x7ff32c19b570 2023-03-27T10:16:22.225 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.573+0000 7ff329ffb700 1 -- 172.21.15.159:0/995539323 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7ff3200196c0 con 0x7ff32c19b570 2023-03-27T10:16:22.225 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.574+0000 7ff333155700 1 -- 172.21.15.159:0/995539323 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff32c10b970 con 0x7ff32c19b570 2023-03-27T10:16:22.226 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.574+0000 7ff329ffb700 1 -- 172.21.15.159:0/995539323 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 9) v1 ==== 43522+0+0 (secure 0 0 0) 0x7ff320017020 con 0x7ff32c19b570 2023-03-27T10:16:22.226 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.575+0000 7ff329ffb700 1 --2- 172.21.15.159:0/995539323 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7ff31c036850 0x7ff31c038d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:22.226 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.575+0000 7ff333155700 1 -- 172.21.15.159:0/995539323 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff32c061e00 con 0x7ff32c19b570 2023-03-27T10:16:22.226 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.575+0000 7ff329ffb700 1 -- 172.21.15.159:0/995539323 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7ff32004a3c0 con 0x7ff32c19b570 2023-03-27T10:16:22.226 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.575+0000 7ff32bfff700 1 -- 172.21.15.159:0/995539323 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7ff31c036850 msgr2=0x7ff31c038d00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.159:6800/3561797228 2023-03-27T10:16:22.227 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.575+0000 7ff32bfff700 1 --2- 172.21.15.159:0/995539323 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7ff31c036850 0x7ff31c038d00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-03-27T10:16:22.227 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.580+0000 7ff329ffb700 1 -- 172.21.15.159:0/995539323 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7ff320010c00 con 0x7ff32c19b570 2023-03-27T10:16:22.227 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.776+0000 7ff32bfff700 1 -- 172.21.15.159:0/995539323 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7ff31c036850 msgr2=0x7ff31c038d00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.159:6800/3561797228 2023-03-27T10:16:22.227 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.776+0000 7ff32bfff700 1 --2- 172.21.15.159:0/995539323 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7ff31c036850 0x7ff31c038d00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2023-03-27T10:16:22.227 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.820+0000 7ff333155700 1 -- 172.21.15.159:0/995539323 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "mgr stat"} v 0) v1 -- 0x7ff32c073bc0 con 0x7ff32c19b570 2023-03-27T10:16:22.227 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.821+0000 7ff329ffb700 1 -- 172.21.15.159:0/995539323 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "mgr stat"}]=0 v9) v1 ==== 56+0+88 (secure 0 0 0) 0x7ff320044af0 con 0x7ff32c19b570 2023-03-27T10:16:22.227 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.823+0000 7ff333155700 1 -- 172.21.15.159:0/995539323 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7ff31c036850 msgr2=0x7ff31c038d00 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-03-27T10:16:22.228 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.823+0000 7ff333155700 1 --2- 172.21.15.159:0/995539323 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7ff31c036850 0x7ff31c038d00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:22.228 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.823+0000 7ff333155700 1 -- 172.21.15.159:0/995539323 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff32c19b570 msgr2=0x7ff32c19b980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:22.228 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.823+0000 7ff333155700 1 --2- 172.21.15.159:0/995539323 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff32c19b570 0x7ff32c19b980 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7ff320003e60 tx=0x7ff320011a30 comp rx=0 tx=0).stop 2023-03-27T10:16:22.228 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.823+0000 7ff333155700 1 -- 172.21.15.159:0/995539323 shutdown_connections 2023-03-27T10:16:22.228 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.823+0000 7ff333155700 1 --2- 172.21.15.159:0/995539323 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7ff31c036850 0x7ff31c038d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:22.228 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.823+0000 7ff333155700 1 --2- 172.21.15.159:0/995539323 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff32c19b570 0x7ff32c19b980 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:22.229 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.823+0000 7ff333155700 1 -- 172.21.15.159:0/995539323 >> 172.21.15.159:0/995539323 conn(0x7ff32c0ffba0 msgr2=0x7ff32c101270 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:22.229 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.823+0000 7ff333155700 1 -- 172.21.15.159:0/995539323 shutdown_connections 2023-03-27T10:16:22.229 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:21.823+0000 7ff333155700 1 -- 172.21.15.159:0/995539323 wait complete. 2023-03-27T10:16:22.229 INFO:teuthology.orchestra.run.smithi159.stdout:Waiting for the mgr to restart... 2023-03-27T10:16:22.229 INFO:teuthology.orchestra.run.smithi159.stdout:Waiting for mgr epoch 9... 2023-03-27T10:16:22.443 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:22 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:22.190+0000 7fc9f3903200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-03-27T10:16:22.785 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:22 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/995539323' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-03-27T10:16:23.114 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:22 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:22.785+0000 7fc9f3903200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-03-27T10:16:23.115 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:22 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:22.954+0000 7fc9f3903200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-03-27T10:16:23.962 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:23 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:23.705+0000 7fc9f3903200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-03-27T10:16:23.963 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:23 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:23.795+0000 7fc9f3903200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-03-27T10:16:23.963 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:23 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:23.963+0000 7fc9f3903200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-03-27T10:16:24.293 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:24 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:24.148+0000 7fc9f3903200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-03-27T10:16:24.614 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:24 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:24.293+0000 7fc9f3903200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-03-27T10:16:25.114 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:24 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:24.689+0000 7fc9f3903200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-03-27T10:16:25.520 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:25 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:25.215+0000 7fc9f3903200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-03-27T10:16:25.822 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:25 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:25.521+0000 7fc9f3903200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-03-27T10:16:25.822 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:25 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:25.708+0000 7fc9f3903200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-03-27T10:16:26.114 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:25 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:25.822+0000 7fc9f3903200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-03-27T10:16:26.115 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:26 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:26.035+0000 7fc9f3903200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-03-27T10:16:27.364 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:16:26 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:16:26.885+0000 7fc9f3903200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-03-27T10:16:28.279 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout { 2023-03-27T10:16:28.280 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-03-27T10:16:28.280 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout "initialized": true 2023-03-27T10:16:28.280 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout } 2023-03-27T10:16:28.280 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.738+0000 7f318b1cf700 1 Processor -- start 2023-03-27T10:16:28.280 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.738+0000 7f318b1cf700 1 -- start start 2023-03-27T10:16:28.281 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.738+0000 7f318b1cf700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3184106150 0x7f3184108530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:28.281 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.738+0000 7f318b1cf700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3184073c40 con 0x7f3184106150 2023-03-27T10:16:28.281 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.738+0000 7f3188f6b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3184106150 0x7f3184108530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:28.281 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.738+0000 7f3188f6b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3184106150 0x7f3184108530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44572/0 (socket says 172.21.15.159:44572) 2023-03-27T10:16:28.281 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.738+0000 7f3188f6b700 1 -- 172.21.15.159:0/2092163876 learned_addr learned my addr 172.21.15.159:0/2092163876 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:28.281 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.739+0000 7f3188f6b700 1 -- 172.21.15.159:0/2092163876 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3184073d80 con 0x7f3184106150 2023-03-27T10:16:28.282 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.739+0000 7f3188f6b700 1 --2- 172.21.15.159:0/2092163876 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3184106150 0x7f3184108530 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f31780098a0 tx=0x7f3178009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f7b02cd13fdff943 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:28.282 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.739+0000 7f31837fe700 1 -- 172.21.15.159:0/2092163876 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f3178004160 con 0x7f3184106150 2023-03-27T10:16:28.282 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.739+0000 7f31837fe700 1 -- 172.21.15.159:0/2092163876 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f31780042c0 con 0x7f3184106150 2023-03-27T10:16:28.282 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.739+0000 7f31837fe700 1 -- 172.21.15.159:0/2092163876 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f31780119b0 con 0x7f3184106150 2023-03-27T10:16:28.282 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.740+0000 7f318b1cf700 1 -- 172.21.15.159:0/2092163876 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3184106150 msgr2=0x7f3184108530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:28.282 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.740+0000 7f318b1cf700 1 --2- 172.21.15.159:0/2092163876 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3184106150 0x7f3184108530 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f31780098a0 tx=0x7f3178009bb0 comp rx=0 tx=0).stop 2023-03-27T10:16:28.282 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.740+0000 7f318b1cf700 1 -- 172.21.15.159:0/2092163876 shutdown_connections 2023-03-27T10:16:28.283 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.740+0000 7f318b1cf700 1 --2- 172.21.15.159:0/2092163876 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3184106150 0x7f3184108530 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:28.283 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.740+0000 7f318b1cf700 1 -- 172.21.15.159:0/2092163876 >> 172.21.15.159:0/2092163876 conn(0x7f31840ffb90 msgr2=0x7f3184101fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:28.283 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.740+0000 7f318b1cf700 1 -- 172.21.15.159:0/2092163876 shutdown_connections 2023-03-27T10:16:28.283 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.740+0000 7f318b1cf700 1 -- 172.21.15.159:0/2092163876 wait complete. 2023-03-27T10:16:28.283 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.740+0000 7f318b1cf700 1 Processor -- start 2023-03-27T10:16:28.283 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.740+0000 7f318b1cf700 1 -- start start 2023-03-27T10:16:28.283 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.740+0000 7f318b1cf700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3184106150 0x7f3184199350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:28.284 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.741+0000 7f318b1cf700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3184199890 con 0x7f3184106150 2023-03-27T10:16:28.284 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.741+0000 7f3188f6b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3184106150 0x7f3184199350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:28.284 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.741+0000 7f3188f6b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3184106150 0x7f3184199350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44574/0 (socket says 172.21.15.159:44574) 2023-03-27T10:16:28.284 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.741+0000 7f3188f6b700 1 -- 172.21.15.159:0/592082107 learned_addr learned my addr 172.21.15.159:0/592082107 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:28.284 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.741+0000 7f3188f6b700 1 -- 172.21.15.159:0/592082107 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3178009550 con 0x7f3184106150 2023-03-27T10:16:28.284 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.741+0000 7f3188f6b700 1 --2- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3184106150 0x7f3184199350 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f3178003e60 tx=0x7f3178011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:28.284 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.741+0000 7f3181ffb700 1 -- 172.21.15.159:0/592082107 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f3178003a60 con 0x7f3184106150 2023-03-27T10:16:28.285 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.741+0000 7f3181ffb700 1 -- 172.21.15.159:0/592082107 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3178003bc0 con 0x7f3184106150 2023-03-27T10:16:28.285 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.741+0000 7f318b1cf700 1 -- 172.21.15.159:0/592082107 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3184109910 con 0x7f3184106150 2023-03-27T10:16:28.285 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.741+0000 7f3181ffb700 1 -- 172.21.15.159:0/592082107 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f31780196c0 con 0x7f3184106150 2023-03-27T10:16:28.285 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.742+0000 7f318b1cf700 1 -- 172.21.15.159:0/592082107 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3184109780 con 0x7f3184106150 2023-03-27T10:16:28.285 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.742+0000 7f3181ffb700 1 -- 172.21.15.159:0/592082107 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 9) v1 ==== 43522+0+0 (secure 0 0 0) 0x7f3178017020 con 0x7f3184106150 2023-03-27T10:16:28.285 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.742+0000 7f3181ffb700 1 --2- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f3174036850 0x7f3174038d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:28.285 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.742+0000 7f3181ffb700 1 -- 172.21.15.159:0/592082107 --> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7f3174039410 con 0x7f3174036850 2023-03-27T10:16:28.286 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.743+0000 7f3183fff700 1 -- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f3174036850 msgr2=0x7f3174038d00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.159:6800/3561797228 2023-03-27T10:16:28.286 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.743+0000 7f3181ffb700 1 -- 172.21.15.159:0/592082107 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f3178049f60 con 0x7f3184106150 2023-03-27T10:16:28.286 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.743+0000 7f3183fff700 1 --2- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f3174036850 0x7f3174038d00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-03-27T10:16:28.286 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.944+0000 7f3183fff700 1 -- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f3174036850 msgr2=0x7f3174038d00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.159:6800/3561797228 2023-03-27T10:16:28.286 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:22.944+0000 7f3183fff700 1 --2- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f3174036850 0x7f3174038d00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2023-03-27T10:16:28.286 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:23.345+0000 7f3183fff700 1 -- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f3174036850 msgr2=0x7f3174038d00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.159:6800/3561797228 2023-03-27T10:16:28.286 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:23.345+0000 7f3183fff700 1 --2- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f3174036850 0x7f3174038d00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2023-03-27T10:16:28.287 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:24.146+0000 7f3183fff700 1 -- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f3174036850 msgr2=0x7f3174038d00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.159:6800/3561797228 2023-03-27T10:16:28.287 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:24.146+0000 7f3183fff700 1 --2- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f3174036850 0x7f3174038d00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2023-03-27T10:16:28.287 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:25.747+0000 7f3183fff700 1 -- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f3174036850 msgr2=0x7f3174038d00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.159:6800/3561797228 2023-03-27T10:16:28.287 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:25.747+0000 7f3183fff700 1 --2- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f3174036850 0x7f3174038d00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 3.200000 2023-03-27T10:16:28.287 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:26.951+0000 7f3181ffb700 1 -- 172.21.15.159:0/592082107 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mgrmap(e 10) v1 ==== 43332+0+0 (secure 0 0 0) 0x7f317800f050 con 0x7f3184106150 2023-03-27T10:16:28.287 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:26.951+0000 7f3181ffb700 1 -- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f3174036850 msgr2=0x7f3174038d00 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-03-27T10:16:28.287 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:26.951+0000 7f3181ffb700 1 --2- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f3174036850 0x7f3174038d00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:28.288 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.953+0000 7f3181ffb700 1 -- 172.21.15.159:0/592082107 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mgrmap(e 11) v1 ==== 43442+0+0 (secure 0 0 0) 0x7f317800ed40 con 0x7f3184106150 2023-03-27T10:16:28.288 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.953+0000 7f3181ffb700 1 --2- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f317403a270 0x7f317403c650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:28.288 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.953+0000 7f3181ffb700 1 -- 172.21.15.159:0/592082107 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7f3174039410 con 0x7f317403a270 2023-03-27T10:16:28.288 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.954+0000 7f3183fff700 1 --2- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f317403a270 0x7f317403c650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:28.288 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.955+0000 7f3183fff700 1 --2- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f317403a270 0x7f317403c650 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7f3170003a10 tx=0x7f31700092b0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:28.288 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.956+0000 7f3181ffb700 1 -- 172.21.15.159:0/592082107 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+6549 (secure 0 0 0) 0x7f3174039410 con 0x7f317403a270 2023-03-27T10:16:28.288 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.962+0000 7f318b1cf700 1 -- 172.21.15.159:0/592082107 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- command(tid 1: {"prefix": "mgr_status"}) v1 -- 0x7f318419a690 con 0x7f317403a270 2023-03-27T10:16:28.289 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.962+0000 7f3181ffb700 1 -- 172.21.15.159:0/592082107 <== mgr.14152 v2:172.21.15.159:6800/3290710483 2 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+52 (secure 0 0 0) 0x7f318419a690 con 0x7f317403a270 2023-03-27T10:16:28.289 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.963+0000 7f318b1cf700 1 -- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f317403a270 msgr2=0x7f317403c650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:28.289 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.963+0000 7f318b1cf700 1 --2- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f317403a270 0x7f317403c650 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7f3170003a10 tx=0x7f31700092b0 comp rx=0 tx=0).stop 2023-03-27T10:16:28.289 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.963+0000 7f318b1cf700 1 -- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3184106150 msgr2=0x7f3184199350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:28.289 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.963+0000 7f318b1cf700 1 --2- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3184106150 0x7f3184199350 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f3178003e60 tx=0x7f3178011a30 comp rx=0 tx=0).stop 2023-03-27T10:16:28.289 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.963+0000 7f318b1cf700 1 -- 172.21.15.159:0/592082107 shutdown_connections 2023-03-27T10:16:28.289 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.963+0000 7f318b1cf700 1 --2- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f317403a270 0x7f317403c650 unknown :-1 s=CLOSED pgs=4 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:28.290 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.963+0000 7f318b1cf700 1 --2- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3184106150 0x7f3184199350 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:28.290 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.963+0000 7f318b1cf700 1 --2- 172.21.15.159:0/592082107 >> [v2:172.21.15.159:6800/3561797228,v1:172.21.15.159:6801/3561797228] conn(0x7f3174036850 0x7f3174038d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:28.290 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.963+0000 7f318b1cf700 1 -- 172.21.15.159:0/592082107 >> 172.21.15.159:0/592082107 conn(0x7f31840ffb90 msgr2=0x7f3184100970 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:28.290 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.963+0000 7f318b1cf700 1 -- 172.21.15.159:0/592082107 shutdown_connections 2023-03-27T10:16:28.290 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:27.963+0000 7f318b1cf700 1 -- 172.21.15.159:0/592082107 wait complete. 2023-03-27T10:16:28.290 INFO:teuthology.orchestra.run.smithi159.stdout:mgr epoch 9 is available 2023-03-27T10:16:28.290 INFO:teuthology.orchestra.run.smithi159.stdout:Generating a dashboard self-signed certificate... 2023-03-27T10:16:28.359 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:27 smithi159 ceph-mon[93039]: Active manager daemon x restarted 2023-03-27T10:16:28.360 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:27 smithi159 ceph-mon[93039]: Activating manager daemon x 2023-03-27T10:16:28.360 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:27 smithi159 ceph-mon[93039]: osdmap e3: 0 total, 0 up, 0 in 2023-03-27T10:16:28.360 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:27 smithi159 ceph-mon[93039]: mgrmap e10: x(active, starting, since 0.052813s) 2023-03-27T10:16:28.360 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:27 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:16:28.360 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:27 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-03-27T10:16:28.360 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:27 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-03-27T10:16:28.360 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:27 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-03-27T10:16:28.361 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:27 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-03-27T10:16:28.361 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:27 smithi159 ceph-mon[93039]: Manager daemon x is now available 2023-03-27T10:16:28.361 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:27 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:16:28.361 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:27 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:16:28.361 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:27 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-03-27T10:16:28.361 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:27 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-03-27T10:16:29.295 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:28 smithi159 ceph-mon[93039]: [27/Mar/2023:10:16:27] ENGINE Bus STARTING 2023-03-27T10:16:29.296 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:28 smithi159 ceph-mon[93039]: [27/Mar/2023:10:16:27] ENGINE Serving on https://172.21.15.159:8765 2023-03-27T10:16:29.296 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:28 smithi159 ceph-mon[93039]: [27/Mar/2023:10:16:27] ENGINE Serving on https://172.21.15.159:7150 2023-03-27T10:16:29.296 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:28 smithi159 ceph-mon[93039]: [27/Mar/2023:10:16:27] ENGINE Bus STARTED 2023-03-27T10:16:29.296 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:28 smithi159 ceph-mon[93039]: mgrmap e11: x(active, since 1.05564s) 2023-03-27T10:16:29.296 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:28 smithi159 ceph-mon[93039]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-03-27T10:16:29.296 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:28 smithi159 ceph-mon[93039]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-03-27T10:16:29.726 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-03-27T10:16:29.727 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.887+0000 7f56f9263700 1 Processor -- start 2023-03-27T10:16:29.727 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.887+0000 7f56f9263700 1 -- start start 2023-03-27T10:16:29.727 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.887+0000 7f56f9263700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f56f4104550 0x7f56f4104960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:29.727 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.887+0000 7f56f9263700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56f4104ea0 con 0x7f56f4104550 2023-03-27T10:16:29.727 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.887+0000 7f56f2ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f56f4104550 0x7f56f4104960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:29.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.887+0000 7f56f2ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f56f4104550 0x7f56f4104960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44630/0 (socket says 172.21.15.159:44630) 2023-03-27T10:16:29.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.887+0000 7f56f2ffd700 1 -- 172.21.15.159:0/1315950981 learned_addr learned my addr 172.21.15.159:0/1315950981 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:29.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.888+0000 7f56f2ffd700 1 -- 172.21.15.159:0/1315950981 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f56f4104fe0 con 0x7f56f4104550 2023-03-27T10:16:29.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.888+0000 7f56f2ffd700 1 --2- 172.21.15.159:0/1315950981 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f56f4104550 0x7f56f4104960 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f56e400bb40 tx=0x7f56e400be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=202e548902a72ee9 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:29.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.888+0000 7f56f1ffb700 1 -- 172.21.15.159:0/1315950981 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f56e400dbe0 con 0x7f56f4104550 2023-03-27T10:16:29.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.888+0000 7f56f1ffb700 1 -- 172.21.15.159:0/1315950981 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f56e40044f0 con 0x7f56f4104550 2023-03-27T10:16:29.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.888+0000 7f56f1ffb700 1 -- 172.21.15.159:0/1315950981 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f56e4003e60 con 0x7f56f4104550 2023-03-27T10:16:29.728 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.888+0000 7f56f9263700 1 -- 172.21.15.159:0/1315950981 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f56f4104550 msgr2=0x7f56f4104960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:29.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.888+0000 7f56f9263700 1 --2- 172.21.15.159:0/1315950981 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f56f4104550 0x7f56f4104960 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f56e400bb40 tx=0x7f56e400be50 comp rx=0 tx=0).stop 2023-03-27T10:16:29.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.889+0000 7f56f9263700 1 -- 172.21.15.159:0/1315950981 shutdown_connections 2023-03-27T10:16:29.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.889+0000 7f56f9263700 1 --2- 172.21.15.159:0/1315950981 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f56f4104550 0x7f56f4104960 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:29.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.889+0000 7f56f9263700 1 -- 172.21.15.159:0/1315950981 >> 172.21.15.159:0/1315950981 conn(0x7f56f40ffba0 msgr2=0x7f56f4101fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:29.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.889+0000 7f56f9263700 1 -- 172.21.15.159:0/1315950981 shutdown_connections 2023-03-27T10:16:29.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.889+0000 7f56f9263700 1 -- 172.21.15.159:0/1315950981 wait complete. 2023-03-27T10:16:29.729 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.889+0000 7f56f9263700 1 Processor -- start 2023-03-27T10:16:29.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.890+0000 7f56f9263700 1 -- start start 2023-03-27T10:16:29.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.890+0000 7f56f9263700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f56f4104550 0x7f56f4197180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:29.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.890+0000 7f56f9263700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56f41976c0 con 0x7f56f4104550 2023-03-27T10:16:29.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.890+0000 7f56f2ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f56f4104550 0x7f56f4197180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:29.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.890+0000 7f56f2ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f56f4104550 0x7f56f4197180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44632/0 (socket says 172.21.15.159:44632) 2023-03-27T10:16:29.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.890+0000 7f56f2ffd700 1 -- 172.21.15.159:0/886697155 learned_addr learned my addr 172.21.15.159:0/886697155 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:29.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.890+0000 7f56f2ffd700 1 -- 172.21.15.159:0/886697155 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f56e400b7f0 con 0x7f56f4104550 2023-03-27T10:16:29.730 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.890+0000 7f56f2ffd700 1 --2- 172.21.15.159:0/886697155 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f56f4104550 0x7f56f4197180 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f56e4003ab0 tx=0x7f56e4003fb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:29.731 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.890+0000 7f56dbfff700 1 -- 172.21.15.159:0/886697155 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f56e401e070 con 0x7f56f4104550 2023-03-27T10:16:29.731 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.890+0000 7f56f9263700 1 -- 172.21.15.159:0/886697155 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f56f41065e0 con 0x7f56f4104550 2023-03-27T10:16:29.731 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.890+0000 7f56dbfff700 1 -- 172.21.15.159:0/886697155 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f56e40109c0 con 0x7f56f4104550 2023-03-27T10:16:29.731 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.890+0000 7f56dbfff700 1 -- 172.21.15.159:0/886697155 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f56e40229a0 con 0x7f56f4104550 2023-03-27T10:16:29.731 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.891+0000 7f56f9263700 1 -- 172.21.15.159:0/886697155 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f56f4106450 con 0x7f56f4104550 2023-03-27T10:16:29.731 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.891+0000 7f56dbfff700 1 -- 172.21.15.159:0/886697155 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 11) v1 ==== 43442+0+0 (secure 0 0 0) 0x7f56e4029030 con 0x7f56f4104550 2023-03-27T10:16:29.731 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.891+0000 7f56f9263700 1 -- 172.21.15.159:0/886697155 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f56f4072e90 con 0x7f56f4104550 2023-03-27T10:16:29.731 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.892+0000 7f56dbfff700 1 --2- 172.21.15.159:0/886697155 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f56dc0367e0 0x7f56dc038c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:29.732 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.892+0000 7f56dbfff700 1 -- 172.21.15.159:0/886697155 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f56e404bf00 con 0x7f56f4104550 2023-03-27T10:16:29.732 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.892+0000 7f56f27fc700 1 --2- 172.21.15.159:0/886697155 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f56dc0367e0 0x7f56dc038c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:29.732 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.892+0000 7f56f27fc700 1 --2- 172.21.15.159:0/886697155 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f56dc0367e0 0x7f56dc038c90 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f56e8006fd0 tx=0x7f56e8006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:29.732 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.896+0000 7f56dbfff700 1 -- 172.21.15.159:0/886697155 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f56e4021590 con 0x7f56f4104550 2023-03-27T10:16:29.732 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:28.957+0000 7f56dbfff700 1 -- 172.21.15.159:0/886697155 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mgrmap(e 12) v1 ==== 43522+0+0 (secure 0 0 0) 0x7f56e4010b30 con 0x7f56f4104550 2023-03-27T10:16:29.732 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:29.100+0000 7f56f9263700 1 -- 172.21.15.159:0/886697155 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}) v1 -- 0x7f56f4060c10 con 0x7f56dc0367e0 2023-03-27T10:16:29.732 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:29.265+0000 7f56dbfff700 1 -- 172.21.15.159:0/886697155 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+32 (secure 0 0 0) 0x7f56f4060c10 con 0x7f56dc0367e0 2023-03-27T10:16:29.733 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:29.267+0000 7f56f9263700 1 -- 172.21.15.159:0/886697155 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f56dc0367e0 msgr2=0x7f56dc038c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:29.733 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:29.267+0000 7f56f9263700 1 --2- 172.21.15.159:0/886697155 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f56dc0367e0 0x7f56dc038c90 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f56e8006fd0 tx=0x7f56e8006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:29.733 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:29.267+0000 7f56f9263700 1 -- 172.21.15.159:0/886697155 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f56f4104550 msgr2=0x7f56f4197180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:29.733 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:29.268+0000 7f56f9263700 1 --2- 172.21.15.159:0/886697155 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f56f4104550 0x7f56f4197180 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f56e4003ab0 tx=0x7f56e4003fb0 comp rx=0 tx=0).stop 2023-03-27T10:16:29.733 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:29.268+0000 7f56f9263700 1 -- 172.21.15.159:0/886697155 shutdown_connections 2023-03-27T10:16:29.733 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:29.268+0000 7f56f9263700 1 --2- 172.21.15.159:0/886697155 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f56dc0367e0 0x7f56dc038c90 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:29.733 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:29.268+0000 7f56f9263700 1 --2- 172.21.15.159:0/886697155 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f56f4104550 0x7f56f4197180 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:29.733 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:29.268+0000 7f56f9263700 1 -- 172.21.15.159:0/886697155 >> 172.21.15.159:0/886697155 conn(0x7f56f40ffba0 msgr2=0x7f56f418dfa0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:29.734 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:29.268+0000 7f56f9263700 1 -- 172.21.15.159:0/886697155 shutdown_connections 2023-03-27T10:16:29.734 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:29.268+0000 7f56f9263700 1 -- 172.21.15.159:0/886697155 wait complete. 2023-03-27T10:16:29.734 INFO:teuthology.orchestra.run.smithi159.stdout:Creating initial admin user... 2023-03-27T10:16:30.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:29 smithi159 ceph-mon[93039]: mgrmap e12: x(active, since 2s) 2023-03-27T10:16:30.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:29 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:30.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:29 smithi159 ceph-mon[93039]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:16:30.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:29 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:30.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:29 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:30.968 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$HmznS/A934ZYN6v.Dc82UePb8QLRtgEnVsGAxahWemxwWqlNAAbOm", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1679912190, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-03-27T10:16:30.968 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.192+0000 7f5c5dc78700 1 Processor -- start 2023-03-27T10:16:30.969 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.192+0000 7f5c5dc78700 1 -- start start 2023-03-27T10:16:30.972 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.192+0000 7f5c5dc78700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5c58104550 0x7f5c58104960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:30.972 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.192+0000 7f5c5dc78700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c58104ea0 con 0x7f5c58104550 2023-03-27T10:16:30.973 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.192+0000 7f5c577fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5c58104550 0x7f5c58104960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:30.973 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.193+0000 7f5c577fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5c58104550 0x7f5c58104960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44636/0 (socket says 172.21.15.159:44636) 2023-03-27T10:16:30.973 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.193+0000 7f5c577fe700 1 -- 172.21.15.159:0/783963893 learned_addr learned my addr 172.21.15.159:0/783963893 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:30.973 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.193+0000 7f5c577fe700 1 -- 172.21.15.159:0/783963893 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5c58104fe0 con 0x7f5c58104550 2023-03-27T10:16:30.973 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.193+0000 7f5c577fe700 1 --2- 172.21.15.159:0/783963893 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5c58104550 0x7f5c58104960 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f5c480098a0 tx=0x7f5c48009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2a4e10d72d2f91d7 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:30.973 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.193+0000 7f5c567fc700 1 -- 172.21.15.159:0/783963893 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5c48004160 con 0x7f5c58104550 2023-03-27T10:16:30.974 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.193+0000 7f5c567fc700 1 -- 172.21.15.159:0/783963893 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5c480042c0 con 0x7f5c58104550 2023-03-27T10:16:30.974 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.193+0000 7f5c567fc700 1 -- 172.21.15.159:0/783963893 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5c480119b0 con 0x7f5c58104550 2023-03-27T10:16:30.974 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.194+0000 7f5c5dc78700 1 -- 172.21.15.159:0/783963893 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5c58104550 msgr2=0x7f5c58104960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:30.974 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.194+0000 7f5c5dc78700 1 --2- 172.21.15.159:0/783963893 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5c58104550 0x7f5c58104960 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f5c480098a0 tx=0x7f5c48009bb0 comp rx=0 tx=0).stop 2023-03-27T10:16:30.974 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.194+0000 7f5c5dc78700 1 -- 172.21.15.159:0/783963893 shutdown_connections 2023-03-27T10:16:30.974 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.194+0000 7f5c5dc78700 1 --2- 172.21.15.159:0/783963893 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5c58104550 0x7f5c58104960 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:30.975 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.194+0000 7f5c5dc78700 1 -- 172.21.15.159:0/783963893 >> 172.21.15.159:0/783963893 conn(0x7f5c580ffba0 msgr2=0x7f5c58101fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:30.975 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.194+0000 7f5c5dc78700 1 -- 172.21.15.159:0/783963893 shutdown_connections 2023-03-27T10:16:30.975 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.194+0000 7f5c5dc78700 1 -- 172.21.15.159:0/783963893 wait complete. 2023-03-27T10:16:30.975 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.194+0000 7f5c5dc78700 1 Processor -- start 2023-03-27T10:16:30.975 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.195+0000 7f5c5dc78700 1 -- start start 2023-03-27T10:16:30.975 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.195+0000 7f5c5dc78700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5c58104550 0x7f5c5818f740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:30.975 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.195+0000 7f5c5dc78700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c5818fc80 con 0x7f5c58104550 2023-03-27T10:16:30.975 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.195+0000 7f5c577fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5c58104550 0x7f5c5818f740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:30.976 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.195+0000 7f5c577fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5c58104550 0x7f5c5818f740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44638/0 (socket says 172.21.15.159:44638) 2023-03-27T10:16:30.976 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.195+0000 7f5c577fe700 1 -- 172.21.15.159:0/866476011 learned_addr learned my addr 172.21.15.159:0/866476011 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:30.976 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.195+0000 7f5c577fe700 1 -- 172.21.15.159:0/866476011 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5c48009550 con 0x7f5c58104550 2023-03-27T10:16:30.976 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.195+0000 7f5c577fe700 1 --2- 172.21.15.159:0/866476011 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5c58104550 0x7f5c5818f740 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f5c48003f00 tx=0x7f5c48011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:30.976 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.195+0000 7f5c54ff9700 1 -- 172.21.15.159:0/866476011 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5c48003b20 con 0x7f5c58104550 2023-03-27T10:16:30.976 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.195+0000 7f5c54ff9700 1 -- 172.21.15.159:0/866476011 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5c48003c80 con 0x7f5c58104550 2023-03-27T10:16:30.977 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.195+0000 7f5c5dc78700 1 -- 172.21.15.159:0/866476011 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5c581065e0 con 0x7f5c58104550 2023-03-27T10:16:30.977 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.195+0000 7f5c54ff9700 1 -- 172.21.15.159:0/866476011 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5c48019770 con 0x7f5c58104550 2023-03-27T10:16:30.977 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.195+0000 7f5c5dc78700 1 -- 172.21.15.159:0/866476011 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5c58106450 con 0x7f5c58104550 2023-03-27T10:16:30.977 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.196+0000 7f5c54ff9700 1 -- 172.21.15.159:0/866476011 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 12) v1 ==== 43522+0+0 (secure 0 0 0) 0x7f5c4802c070 con 0x7f5c58104550 2023-03-27T10:16:30.977 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.196+0000 7f5c54ff9700 1 --2- 172.21.15.159:0/866476011 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f5c400368c0 0x7f5c40038d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:30.977 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.196+0000 7f5c54ff9700 1 -- 172.21.15.159:0/866476011 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f5c48049f10 con 0x7f5c58104550 2023-03-27T10:16:30.977 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.197+0000 7f5c56ffd700 1 --2- 172.21.15.159:0/866476011 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f5c400368c0 0x7f5c40038d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:30.977 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.197+0000 7f5c5dc78700 1 -- 172.21.15.159:0/866476011 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5c58108a20 con 0x7f5c58104550 2023-03-27T10:16:30.978 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.197+0000 7f5c56ffd700 1 --2- 172.21.15.159:0/866476011 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f5c400368c0 0x7f5c40038d70 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f5c4c006fd0 tx=0x7f5c4c006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:30.978 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.202+0000 7f5c54ff9700 1 -- 172.21.15.159:0/866476011 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f5c4800b3e0 con 0x7f5c58104550 2023-03-27T10:16:30.978 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.418+0000 7f5c5dc78700 1 -- 172.21.15.159:0/866476011 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}) v1 -- 0x7f5c58061c20 con 0x7f5c400368c0 2023-03-27T10:16:30.978 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.643+0000 7f5c54ff9700 1 -- 172.21.15.159:0/866476011 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+252 (secure 0 0 0) 0x7f5c58061c20 con 0x7f5c400368c0 2023-03-27T10:16:30.978 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.645+0000 7f5c5dc78700 1 -- 172.21.15.159:0/866476011 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f5c400368c0 msgr2=0x7f5c40038d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:30.978 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.645+0000 7f5c5dc78700 1 --2- 172.21.15.159:0/866476011 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f5c400368c0 0x7f5c40038d70 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f5c4c006fd0 tx=0x7f5c4c006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:30.979 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.645+0000 7f5c5dc78700 1 -- 172.21.15.159:0/866476011 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5c58104550 msgr2=0x7f5c5818f740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:30.979 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.645+0000 7f5c5dc78700 1 --2- 172.21.15.159:0/866476011 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5c58104550 0x7f5c5818f740 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f5c48003f00 tx=0x7f5c48011a30 comp rx=0 tx=0).stop 2023-03-27T10:16:30.979 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.645+0000 7f5c5dc78700 1 -- 172.21.15.159:0/866476011 shutdown_connections 2023-03-27T10:16:30.979 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.645+0000 7f5c5dc78700 1 --2- 172.21.15.159:0/866476011 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f5c400368c0 0x7f5c40038d70 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:30.979 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.645+0000 7f5c5dc78700 1 --2- 172.21.15.159:0/866476011 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5c58104550 0x7f5c5818f740 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:30.979 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.646+0000 7f5c5dc78700 1 -- 172.21.15.159:0/866476011 >> 172.21.15.159:0/866476011 conn(0x7f5c580ffba0 msgr2=0x7f5c58073800 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:30.979 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.646+0000 7f5c5dc78700 1 -- 172.21.15.159:0/866476011 shutdown_connections 2023-03-27T10:16:30.979 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:30.646+0000 7f5c5dc78700 1 -- 172.21.15.159:0/866476011 wait complete. 2023-03-27T10:16:30.980 INFO:teuthology.orchestra.run.smithi159.stdout:Fetching dashboard port number... 2023-03-27T10:16:31.270 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:31.270 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:30 smithi159 ceph-mon[93039]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:16:31.270 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:32.074 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:31 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/2551713962' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-03-27T10:16:32.283 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stdout 8443 2023-03-27T10:16:32.283 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.686+0000 7f7d2005c700 1 Processor -- start 2023-03-27T10:16:32.283 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.686+0000 7f7d2005c700 1 -- start start 2023-03-27T10:16:32.283 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.687+0000 7f7d2005c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d181056c0 0x7f7d18105ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:32.284 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.687+0000 7f7d2005c700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d18106010 con 0x7f7d181056c0 2023-03-27T10:16:32.284 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.687+0000 7f7d1ddf8700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d181056c0 0x7f7d18105ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:32.284 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.687+0000 7f7d1ddf8700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d181056c0 0x7f7d18105ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44642/0 (socket says 172.21.15.159:44642) 2023-03-27T10:16:32.284 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.687+0000 7f7d1ddf8700 1 -- 172.21.15.159:0/1610852183 learned_addr learned my addr 172.21.15.159:0/1610852183 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:32.284 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.687+0000 7f7d1ddf8700 1 -- 172.21.15.159:0/1610852183 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d18106150 con 0x7f7d181056c0 2023-03-27T10:16:32.284 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.687+0000 7f7d1ddf8700 1 --2- 172.21.15.159:0/1610852183 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d181056c0 0x7f7d18105ad0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f7d0c0098a0 tx=0x7f7d0c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4a2eef6af616489f server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:32.284 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.687+0000 7f7d1cdf6700 1 -- 172.21.15.159:0/1610852183 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7d0c004160 con 0x7f7d181056c0 2023-03-27T10:16:32.285 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.687+0000 7f7d1cdf6700 1 -- 172.21.15.159:0/1610852183 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7d0c0042c0 con 0x7f7d181056c0 2023-03-27T10:16:32.285 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.687+0000 7f7d1cdf6700 1 -- 172.21.15.159:0/1610852183 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7d0c0119b0 con 0x7f7d181056c0 2023-03-27T10:16:32.285 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.688+0000 7f7d2005c700 1 -- 172.21.15.159:0/1610852183 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d181056c0 msgr2=0x7f7d18105ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:32.285 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.688+0000 7f7d2005c700 1 --2- 172.21.15.159:0/1610852183 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d181056c0 0x7f7d18105ad0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f7d0c0098a0 tx=0x7f7d0c009bb0 comp rx=0 tx=0).stop 2023-03-27T10:16:32.285 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.688+0000 7f7d2005c700 1 -- 172.21.15.159:0/1610852183 shutdown_connections 2023-03-27T10:16:32.285 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.688+0000 7f7d2005c700 1 --2- 172.21.15.159:0/1610852183 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d181056c0 0x7f7d18105ad0 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:32.286 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.688+0000 7f7d2005c700 1 -- 172.21.15.159:0/1610852183 >> 172.21.15.159:0/1610852183 conn(0x7f7d18100d10 msgr2=0x7f7d18103140 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:32.286 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.688+0000 7f7d2005c700 1 -- 172.21.15.159:0/1610852183 shutdown_connections 2023-03-27T10:16:32.286 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.688+0000 7f7d2005c700 1 -- 172.21.15.159:0/1610852183 wait complete. 2023-03-27T10:16:32.286 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.689+0000 7f7d2005c700 1 Processor -- start 2023-03-27T10:16:32.286 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.689+0000 7f7d2005c700 1 -- start start 2023-03-27T10:16:32.286 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.689+0000 7f7d2005c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d181056c0 0x7f7d181992e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:32.286 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.689+0000 7f7d2005c700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d18199820 con 0x7f7d181056c0 2023-03-27T10:16:32.287 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.689+0000 7f7d1ddf8700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d181056c0 0x7f7d181992e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:32.287 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.689+0000 7f7d1ddf8700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d181056c0 0x7f7d181992e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44644/0 (socket says 172.21.15.159:44644) 2023-03-27T10:16:32.287 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.689+0000 7f7d1ddf8700 1 -- 172.21.15.159:0/2551713962 learned_addr learned my addr 172.21.15.159:0/2551713962 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:32.287 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.689+0000 7f7d1ddf8700 1 -- 172.21.15.159:0/2551713962 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d0c009550 con 0x7f7d181056c0 2023-03-27T10:16:32.287 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.689+0000 7f7d1ddf8700 1 --2- 172.21.15.159:0/2551713962 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d181056c0 0x7f7d181992e0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f7d0c011980 tx=0x7f7d0c011a60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:32.287 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.690+0000 7f7d0affd700 1 -- 172.21.15.159:0/2551713962 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7d0c003a60 con 0x7f7d181056c0 2023-03-27T10:16:32.287 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.690+0000 7f7d0affd700 1 -- 172.21.15.159:0/2551713962 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7d0c003bc0 con 0x7f7d181056c0 2023-03-27T10:16:32.287 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.690+0000 7f7d2005c700 1 -- 172.21.15.159:0/2551713962 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7d18107750 con 0x7f7d181056c0 2023-03-27T10:16:32.288 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.690+0000 7f7d0affd700 1 -- 172.21.15.159:0/2551713962 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7d0c01a670 con 0x7f7d181056c0 2023-03-27T10:16:32.288 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.690+0000 7f7d2005c700 1 -- 172.21.15.159:0/2551713962 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7d181075c0 con 0x7f7d181056c0 2023-03-27T10:16:32.288 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.690+0000 7f7d0affd700 1 -- 172.21.15.159:0/2551713962 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 12) v1 ==== 43522+0+0 (secure 0 0 0) 0x7f7d0c017020 con 0x7f7d181056c0 2023-03-27T10:16:32.288 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.691+0000 7f7d0affd700 1 --2- 172.21.15.159:0/2551713962 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7d04036870 0x7f7d04038d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:32.288 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.691+0000 7f7d0affd700 1 -- 172.21.15.159:0/2551713962 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f7d0c049d70 con 0x7f7d181056c0 2023-03-27T10:16:32.288 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.691+0000 7f7d2005c700 1 -- 172.21.15.159:0/2551713962 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7d18109b90 con 0x7f7d181056c0 2023-03-27T10:16:32.289 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.691+0000 7f7d1d5f7700 1 --2- 172.21.15.159:0/2551713962 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7d04036870 0x7f7d04038d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:32.289 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.691+0000 7f7d1d5f7700 1 --2- 172.21.15.159:0/2551713962 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7d04036870 0x7f7d04038d20 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f7d14006fd0 tx=0x7f7d14006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:32.289 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.695+0000 7f7d0affd700 1 -- 172.21.15.159:0/2551713962 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f7d0c01a930 con 0x7f7d181056c0 2023-03-27T10:16:32.289 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.886+0000 7f7d2005c700 1 -- 172.21.15.159:0/2551713962 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"} v 0) v1 -- 0x7f7d18061e00 con 0x7f7d181056c0 2023-03-27T10:16:32.289 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.886+0000 7f7d0affd700 1 -- 172.21.15.159:0/2551713962 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]=0 v8) v1 ==== 112+0+5 (secure 0 0 0) 0x7f7d0c018350 con 0x7f7d181056c0 2023-03-27T10:16:32.289 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.888+0000 7f7d2005c700 1 -- 172.21.15.159:0/2551713962 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7d04036870 msgr2=0x7f7d04038d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:32.289 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.888+0000 7f7d2005c700 1 --2- 172.21.15.159:0/2551713962 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7d04036870 0x7f7d04038d20 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f7d14006fd0 tx=0x7f7d14006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:32.290 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.888+0000 7f7d2005c700 1 -- 172.21.15.159:0/2551713962 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d181056c0 msgr2=0x7f7d181992e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:32.290 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.888+0000 7f7d2005c700 1 --2- 172.21.15.159:0/2551713962 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d181056c0 0x7f7d181992e0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f7d0c011980 tx=0x7f7d0c011a60 comp rx=0 tx=0).stop 2023-03-27T10:16:32.290 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.888+0000 7f7d2005c700 1 -- 172.21.15.159:0/2551713962 shutdown_connections 2023-03-27T10:16:32.290 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.888+0000 7f7d2005c700 1 --2- 172.21.15.159:0/2551713962 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7d04036870 0x7f7d04038d20 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:32.290 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.888+0000 7f7d2005c700 1 --2- 172.21.15.159:0/2551713962 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d181056c0 0x7f7d181992e0 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:32.290 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.888+0000 7f7d2005c700 1 -- 172.21.15.159:0/2551713962 >> 172.21.15.159:0/2551713962 conn(0x7f7d18100d10 msgr2=0x7f7d18101a10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:32.290 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.888+0000 7f7d2005c700 1 -- 172.21.15.159:0/2551713962 shutdown_connections 2023-03-27T10:16:32.291 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:31.888+0000 7f7d2005c700 1 -- 172.21.15.159:0/2551713962 wait complete. 2023-03-27T10:16:32.291 INFO:teuthology.orchestra.run.smithi159.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-03-27T10:16:32.292 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stdout disabled 2023-03-27T10:16:32.299 INFO:teuthology.orchestra.run.smithi159.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-03-27T10:16:32.299 INFO:teuthology.orchestra.run.smithi159.stdout:systemctl: stdout inactive 2023-03-27T10:16:32.300 INFO:teuthology.orchestra.run.smithi159.stdout:firewalld.service is not enabled 2023-03-27T10:16:32.300 INFO:teuthology.orchestra.run.smithi159.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-03-27T10:16:32.301 INFO:teuthology.orchestra.run.smithi159.stdout:Ceph Dashboard is now available at: 2023-03-27T10:16:32.301 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:16:32.301 INFO:teuthology.orchestra.run.smithi159.stdout: URL: https://smithi159.front.sepia.ceph.com:8443/ 2023-03-27T10:16:32.301 INFO:teuthology.orchestra.run.smithi159.stdout: User: admin 2023-03-27T10:16:32.302 INFO:teuthology.orchestra.run.smithi159.stdout: Password: omgv4o0axf 2023-03-27T10:16:32.302 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:16:32.302 INFO:teuthology.orchestra.run.smithi159.stdout:Saving cluster configuration to /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config directory 2023-03-27T10:16:32.303 INFO:teuthology.orchestra.run.smithi159.stdout:Enabling autotune for osd_memory_target 2023-03-27T10:16:33.346 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.762+0000 7fdd86456700 1 Processor -- start 2023-03-27T10:16:33.347 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.762+0000 7fdd86456700 1 -- start start 2023-03-27T10:16:33.347 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.762+0000 7fdd86456700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fdd80104540 0x7fdd80104950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:33.347 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.762+0000 7fdd86456700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd80104e90 con 0x7fdd80104540 2023-03-27T10:16:33.347 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.762+0000 7fdd7ffff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fdd80104540 0x7fdd80104950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:33.347 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.762+0000 7fdd7ffff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fdd80104540 0x7fdd80104950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44652/0 (socket says 172.21.15.159:44652) 2023-03-27T10:16:33.347 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.763+0000 7fdd7ffff700 1 -- 172.21.15.159:0/2998823649 learned_addr learned my addr 172.21.15.159:0/2998823649 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:33.347 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.763+0000 7fdd7ffff700 1 -- 172.21.15.159:0/2998823649 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdd80104fd0 con 0x7fdd80104540 2023-03-27T10:16:33.347 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.763+0000 7fdd7ffff700 1 --2- 172.21.15.159:0/2998823649 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fdd80104540 0x7fdd80104950 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7fdd700098a0 tx=0x7fdd70009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=10f73fbb6d17bf0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:33.348 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.763+0000 7fdd7effd700 1 -- 172.21.15.159:0/2998823649 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fdd70004160 con 0x7fdd80104540 2023-03-27T10:16:33.348 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.763+0000 7fdd7effd700 1 -- 172.21.15.159:0/2998823649 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdd700042c0 con 0x7fdd80104540 2023-03-27T10:16:33.348 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.763+0000 7fdd7effd700 1 -- 172.21.15.159:0/2998823649 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fdd700119b0 con 0x7fdd80104540 2023-03-27T10:16:33.348 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.764+0000 7fdd86456700 1 -- 172.21.15.159:0/2998823649 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fdd80104540 msgr2=0x7fdd80104950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:33.348 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.764+0000 7fdd86456700 1 --2- 172.21.15.159:0/2998823649 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fdd80104540 0x7fdd80104950 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7fdd700098a0 tx=0x7fdd70009bb0 comp rx=0 tx=0).stop 2023-03-27T10:16:33.348 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.764+0000 7fdd86456700 1 -- 172.21.15.159:0/2998823649 shutdown_connections 2023-03-27T10:16:33.348 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.764+0000 7fdd86456700 1 --2- 172.21.15.159:0/2998823649 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fdd80104540 0x7fdd80104950 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:33.349 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.764+0000 7fdd86456700 1 -- 172.21.15.159:0/2998823649 >> 172.21.15.159:0/2998823649 conn(0x7fdd800ffb90 msgr2=0x7fdd80101fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:33.349 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.764+0000 7fdd86456700 1 -- 172.21.15.159:0/2998823649 shutdown_connections 2023-03-27T10:16:33.349 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.764+0000 7fdd86456700 1 -- 172.21.15.159:0/2998823649 wait complete. 2023-03-27T10:16:33.349 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.764+0000 7fdd86456700 1 Processor -- start 2023-03-27T10:16:33.349 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.764+0000 7fdd86456700 1 -- start start 2023-03-27T10:16:33.349 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.765+0000 7fdd86456700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fdd80104540 0x7fdd80073a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:33.349 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.765+0000 7fdd86456700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd8019c010 con 0x7fdd80104540 2023-03-27T10:16:33.350 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.765+0000 7fdd7ffff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fdd80104540 0x7fdd80073a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:33.350 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.765+0000 7fdd7ffff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fdd80104540 0x7fdd80073a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44654/0 (socket says 172.21.15.159:44654) 2023-03-27T10:16:33.350 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.765+0000 7fdd7ffff700 1 -- 172.21.15.159:0/423208236 learned_addr learned my addr 172.21.15.159:0/423208236 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:33.350 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.765+0000 7fdd7ffff700 1 -- 172.21.15.159:0/423208236 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdd70009550 con 0x7fdd80104540 2023-03-27T10:16:33.350 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.765+0000 7fdd7ffff700 1 --2- 172.21.15.159:0/423208236 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fdd80104540 0x7fdd80073a50 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fdd70003f00 tx=0x7fdd70011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:33.350 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.765+0000 7fdd7d7fa700 1 -- 172.21.15.159:0/423208236 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fdd70003b20 con 0x7fdd80104540 2023-03-27T10:16:33.350 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.765+0000 7fdd7d7fa700 1 -- 172.21.15.159:0/423208236 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdd70003c80 con 0x7fdd80104540 2023-03-27T10:16:33.350 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.765+0000 7fdd7d7fa700 1 -- 172.21.15.159:0/423208236 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fdd70019770 con 0x7fdd80104540 2023-03-27T10:16:33.351 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.765+0000 7fdd86456700 1 -- 172.21.15.159:0/423208236 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdd801065d0 con 0x7fdd80104540 2023-03-27T10:16:33.351 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.765+0000 7fdd86456700 1 -- 172.21.15.159:0/423208236 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdd80106440 con 0x7fdd80104540 2023-03-27T10:16:33.351 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.766+0000 7fdd7d7fa700 1 -- 172.21.15.159:0/423208236 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 12) v1 ==== 43522+0+0 (secure 0 0 0) 0x7fdd7002c070 con 0x7fdd80104540 2023-03-27T10:16:33.351 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.766+0000 7fdd7d7fa700 1 --2- 172.21.15.159:0/423208236 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fdd680368c0 0x7fdd68038d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:33.351 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.766+0000 7fdd86456700 1 -- 172.21.15.159:0/423208236 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdd80061e00 con 0x7fdd80104540 2023-03-27T10:16:33.351 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.766+0000 7fdd7d7fa700 1 -- 172.21.15.159:0/423208236 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fdd70049ee0 con 0x7fdd80104540 2023-03-27T10:16:33.351 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.766+0000 7fdd7f7fe700 1 --2- 172.21.15.159:0/423208236 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fdd680368c0 0x7fdd68038d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:33.351 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.767+0000 7fdd7f7fe700 1 --2- 172.21.15.159:0/423208236 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fdd680368c0 0x7fdd68038d70 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7fdd74006fd0 tx=0x7fdd74006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:33.352 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.771+0000 7fdd7d7fa700 1 -- 172.21.15.159:0/423208236 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fdd7000b3e0 con 0x7fdd80104540 2023-03-27T10:16:33.352 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.953+0000 7fdd86456700 1 -- 172.21.15.159:0/423208236 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command([{prefix=config set, name=osd_memory_target_autotune}] v 0) v1 -- 0x7fdd8019c8b0 con 0x7fdd80104540 2023-03-27T10:16:33.352 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.953+0000 7fdd7d7fa700 1 -- 172.21.15.159:0/423208236 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{prefix=config set, name=osd_memory_target_autotune}]=0 v8)=0 v8) v1 ==== 127+0+0 (secure 0 0 0) 0x7fdd70018350 con 0x7fdd80104540 2023-03-27T10:16:33.352 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.955+0000 7fdd86456700 1 -- 172.21.15.159:0/423208236 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fdd680368c0 msgr2=0x7fdd68038d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:33.352 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.955+0000 7fdd86456700 1 --2- 172.21.15.159:0/423208236 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fdd680368c0 0x7fdd68038d70 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7fdd74006fd0 tx=0x7fdd74006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:33.352 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.955+0000 7fdd86456700 1 -- 172.21.15.159:0/423208236 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fdd80104540 msgr2=0x7fdd80073a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:33.352 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.955+0000 7fdd86456700 1 --2- 172.21.15.159:0/423208236 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fdd80104540 0x7fdd80073a50 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fdd70003f00 tx=0x7fdd70011a30 comp rx=0 tx=0).stop 2023-03-27T10:16:33.353 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.956+0000 7fdd86456700 1 -- 172.21.15.159:0/423208236 shutdown_connections 2023-03-27T10:16:33.353 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.956+0000 7fdd86456700 1 --2- 172.21.15.159:0/423208236 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fdd680368c0 0x7fdd68038d70 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:33.353 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.956+0000 7fdd86456700 1 --2- 172.21.15.159:0/423208236 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fdd80104540 0x7fdd80073a50 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:33.353 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.956+0000 7fdd86456700 1 -- 172.21.15.159:0/423208236 >> 172.21.15.159:0/423208236 conn(0x7fdd800ffb90 msgr2=0x7fdd801008e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:33.353 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.956+0000 7fdd86456700 1 -- 172.21.15.159:0/423208236 shutdown_connections 2023-03-27T10:16:33.353 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:32.956+0000 7fdd86456700 1 -- 172.21.15.159:0/423208236 wait complete. 2023-03-27T10:16:34.253 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:33 smithi159 ceph-mon[93039]: mgrmap e13: x(active, since 6s) 2023-03-27T10:16:34.491 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.841+0000 7f7099d59700 1 Processor -- start 2023-03-27T10:16:34.491 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.841+0000 7f7099d59700 1 -- start start 2023-03-27T10:16:34.491 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.842+0000 7f7099d59700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7094104540 0x7f7094104950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:34.491 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.842+0000 7f7099d59700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7094104e90 con 0x7f7094104540 2023-03-27T10:16:34.492 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.842+0000 7f70937fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7094104540 0x7f7094104950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:34.492 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.842+0000 7f70937fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7094104540 0x7f7094104950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44658/0 (socket says 172.21.15.159:44658) 2023-03-27T10:16:34.492 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.842+0000 7f70937fe700 1 -- 172.21.15.159:0/3838890845 learned_addr learned my addr 172.21.15.159:0/3838890845 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:34.492 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.842+0000 7f70937fe700 1 -- 172.21.15.159:0/3838890845 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7094104fd0 con 0x7f7094104540 2023-03-27T10:16:34.492 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.842+0000 7f70937fe700 1 --2- 172.21.15.159:0/3838890845 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7094104540 0x7f7094104950 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f70840098a0 tx=0x7f7084009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6d7d612c4ef994eb server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:34.493 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.842+0000 7f70927fc700 1 -- 172.21.15.159:0/3838890845 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7084004160 con 0x7f7094104540 2023-03-27T10:16:34.493 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.842+0000 7f70927fc700 1 -- 172.21.15.159:0/3838890845 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f70840042c0 con 0x7f7094104540 2023-03-27T10:16:34.493 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.842+0000 7f70927fc700 1 -- 172.21.15.159:0/3838890845 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f70840119b0 con 0x7f7094104540 2023-03-27T10:16:34.493 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.843+0000 7f7099d59700 1 -- 172.21.15.159:0/3838890845 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7094104540 msgr2=0x7f7094104950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:34.493 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.843+0000 7f7099d59700 1 --2- 172.21.15.159:0/3838890845 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7094104540 0x7f7094104950 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f70840098a0 tx=0x7f7084009bb0 comp rx=0 tx=0).stop 2023-03-27T10:16:34.493 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.843+0000 7f7099d59700 1 -- 172.21.15.159:0/3838890845 shutdown_connections 2023-03-27T10:16:34.494 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.843+0000 7f7099d59700 1 --2- 172.21.15.159:0/3838890845 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7094104540 0x7f7094104950 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:34.494 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.843+0000 7f7099d59700 1 -- 172.21.15.159:0/3838890845 >> 172.21.15.159:0/3838890845 conn(0x7f70940ffb90 msgr2=0x7f7094101fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:34.494 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.843+0000 7f7099d59700 1 -- 172.21.15.159:0/3838890845 shutdown_connections 2023-03-27T10:16:34.494 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.843+0000 7f7099d59700 1 -- 172.21.15.159:0/3838890845 wait complete. 2023-03-27T10:16:34.494 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.844+0000 7f7099d59700 1 Processor -- start 2023-03-27T10:16:34.494 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.844+0000 7f7099d59700 1 -- start start 2023-03-27T10:16:34.494 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.844+0000 7f7099d59700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7094104540 0x7f7094073a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:34.495 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.844+0000 7f7099d59700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f709419c010 con 0x7f7094104540 2023-03-27T10:16:34.495 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.844+0000 7f70937fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7094104540 0x7f7094073a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:34.495 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.844+0000 7f70937fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7094104540 0x7f7094073a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44660/0 (socket says 172.21.15.159:44660) 2023-03-27T10:16:34.495 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.844+0000 7f70937fe700 1 -- 172.21.15.159:0/1770194536 learned_addr learned my addr 172.21.15.159:0/1770194536 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:34.495 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.845+0000 7f70937fe700 1 -- 172.21.15.159:0/1770194536 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7084009550 con 0x7f7094104540 2023-03-27T10:16:34.495 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.845+0000 7f70937fe700 1 --2- 172.21.15.159:0/1770194536 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7094104540 0x7f7094073a50 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f7084006af0 tx=0x7f7084011a10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:34.495 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.845+0000 7f7090ff9700 1 -- 172.21.15.159:0/1770194536 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7084003a40 con 0x7f7094104540 2023-03-27T10:16:34.496 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.845+0000 7f7090ff9700 1 -- 172.21.15.159:0/1770194536 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7084003ba0 con 0x7f7094104540 2023-03-27T10:16:34.496 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.845+0000 7f7090ff9700 1 -- 172.21.15.159:0/1770194536 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f70840196c0 con 0x7f7094104540 2023-03-27T10:16:34.496 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.845+0000 7f7099d59700 1 -- 172.21.15.159:0/1770194536 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f70941065d0 con 0x7f7094104540 2023-03-27T10:16:34.496 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.845+0000 7f7099d59700 1 -- 172.21.15.159:0/1770194536 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7094106440 con 0x7f7094104540 2023-03-27T10:16:34.496 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.846+0000 7f7090ff9700 1 -- 172.21.15.159:0/1770194536 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 13) v1 ==== 43566+0+0 (secure 0 0 0) 0x7f708402a020 con 0x7f7094104540 2023-03-27T10:16:34.496 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.846+0000 7f7090ff9700 1 --2- 172.21.15.159:0/1770194536 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f707c036900 0x7f707c038db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:34.496 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.846+0000 7f7090ff9700 1 -- 172.21.15.159:0/1770194536 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f708404a020 con 0x7f7094104540 2023-03-27T10:16:34.497 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.846+0000 7f7099d59700 1 -- 172.21.15.159:0/1770194536 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7094061e00 con 0x7f7094104540 2023-03-27T10:16:34.497 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.846+0000 7f7092ffd700 1 --2- 172.21.15.159:0/1770194536 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f707c036900 0x7f707c038db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:34.497 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.846+0000 7f7092ffd700 1 --2- 172.21.15.159:0/1770194536 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f707c036900 0x7f707c038db0 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f7088006fd0 tx=0x7f7088006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:34.497 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:33.851+0000 7f7090ff9700 1 -- 172.21.15.159:0/1770194536 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f7084019930 con 0x7f7094104540 2023-03-27T10:16:34.497 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:34.133+0000 7f7099d59700 1 -- 172.21.15.159:0/1770194536 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command([{prefix=config-key set, key=mgr/dashboard/cluster/status}] v 0) v1 -- 0x7f709419c9a0 con 0x7f7094104540 2023-03-27T10:16:34.497 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:34.136+0000 7f7090ff9700 1 -- 172.21.15.159:0/1770194536 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{prefix=config-key set, key=mgr/dashboard/cluster/status}]=0 set mgr/dashboard/cluster/status v30)=0 set mgr/dashboard/cluster/status v30) v1 ==== 153+0+0 (secure 0 0 0) 0x7f7084047020 con 0x7f7094104540 2023-03-27T10:16:34.497 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:34.138+0000 7f7099d59700 1 -- 172.21.15.159:0/1770194536 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f707c036900 msgr2=0x7f707c038db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:34.498 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:34.138+0000 7f7099d59700 1 --2- 172.21.15.159:0/1770194536 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f707c036900 0x7f707c038db0 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f7088006fd0 tx=0x7f7088006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:34.498 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:34.138+0000 7f7099d59700 1 -- 172.21.15.159:0/1770194536 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7094104540 msgr2=0x7f7094073a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:34.498 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:34.138+0000 7f7099d59700 1 --2- 172.21.15.159:0/1770194536 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7094104540 0x7f7094073a50 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f7084006af0 tx=0x7f7084011a10 comp rx=0 tx=0).stop 2023-03-27T10:16:34.498 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:34.138+0000 7f7099d59700 1 -- 172.21.15.159:0/1770194536 shutdown_connections 2023-03-27T10:16:34.498 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:34.138+0000 7f7099d59700 1 --2- 172.21.15.159:0/1770194536 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f707c036900 0x7f707c038db0 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:34.498 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:34.138+0000 7f7099d59700 1 --2- 172.21.15.159:0/1770194536 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7094104540 0x7f7094073a50 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:34.498 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:34.138+0000 7f7099d59700 1 -- 172.21.15.159:0/1770194536 >> 172.21.15.159:0/1770194536 conn(0x7f70940ffb90 msgr2=0x7f70941008e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:34.499 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:34.138+0000 7f7099d59700 1 -- 172.21.15.159:0/1770194536 shutdown_connections 2023-03-27T10:16:34.499 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr 2023-03-27T10:16:34.138+0000 7f7099d59700 1 -- 172.21.15.159:0/1770194536 wait complete. 2023-03-27T10:16:34.499 INFO:teuthology.orchestra.run.smithi159.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-03-27T10:16:34.499 INFO:teuthology.orchestra.run.smithi159.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-03-27T10:16:34.499 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:16:34.499 INFO:teuthology.orchestra.run.smithi159.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-03-27T10:16:34.499 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:16:34.500 INFO:teuthology.orchestra.run.smithi159.stdout:Or, if you are only running a single cluster on this host: 2023-03-27T10:16:34.500 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:16:34.500 INFO:teuthology.orchestra.run.smithi159.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-03-27T10:16:34.500 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:16:34.500 INFO:teuthology.orchestra.run.smithi159.stdout:Please consider enabling telemetry to help improve Ceph: 2023-03-27T10:16:34.500 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:16:34.500 INFO:teuthology.orchestra.run.smithi159.stdout: ceph telemetry on 2023-03-27T10:16:34.501 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:16:34.501 INFO:teuthology.orchestra.run.smithi159.stdout:For more information see: 2023-03-27T10:16:34.501 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:16:34.501 INFO:teuthology.orchestra.run.smithi159.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2023-03-27T10:16:34.501 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:16:34.501 INFO:teuthology.orchestra.run.smithi159.stdout:Bootstrap complete. 2023-03-27T10:16:34.524 INFO:tasks.cephadm:Fetching config... 2023-03-27T10:16:34.525 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2023-03-27T10:16:34.525 DEBUG:teuthology.orchestra.run.smithi159:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-03-27T10:16:34.542 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-03-27T10:16:34.542 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2023-03-27T10:16:34.543 DEBUG:teuthology.orchestra.run.smithi159:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-03-27T10:16:34.598 INFO:tasks.cephadm:Fetching mon keyring... 2023-03-27T10:16:34.599 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2023-03-27T10:16:34.599 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd if=/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/keyring of=/dev/stdout 2023-03-27T10:16:34.668 INFO:tasks.cephadm:Fetching pub ssh key... 2023-03-27T10:16:34.668 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2023-03-27T10:16:34.669 DEBUG:teuthology.orchestra.run.smithi159:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-03-27T10:16:34.724 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-03-27T10:16:34.725 DEBUG:teuthology.orchestra.run.smithi159:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx1sQLTEmyAgXsE9t6c87JfjiJetYiuI+ur7emKZG6fw1dcawIFy68YejeEc52YSsc93Ez/ySp37R2DbAdyLlqFqiEyKL6VVT+gzzKenY3MpodS7C4NpWthSxkK8HlOGxzWklb8T5GhI+ZB3xb3Yp08Jej8lGQf7xPldxpjlmdOpnKdFQqmuQBgeLEljN3cHlfzd9KHSBrF1g0VgBsX6cRUE2Kx8rmOFpsVn7VDH9u4vDHb3poS1V19NXOxoLsvHh1O6PJsxLgwNwtPOdXxVoIiXqXL81cOjI7UoMoisq3IwcsvWIV7a2Ngbg0nxP/D0+TjyTpyP7aWt4esL6AyKZiMr0snTCqW8HGzLwJ7fDH+4Gama60NlUs497uzJ44o/DP6XBozhn/H2l5+pTBKDNCoRiInszpw7kLXxGunmEH7azHmWljTaIId80HVaN/fWZgGY/xZRE6JNqeZNfX2uNSY4KG4C/FqEcc9hBxfuDCdi9cNQk+k7vE9MmNHNwwx2s= ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-03-27T10:16:34.800 INFO:teuthology.orchestra.run.smithi159.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx1sQLTEmyAgXsE9t6c87JfjiJetYiuI+ur7emKZG6fw1dcawIFy68YejeEc52YSsc93Ez/ySp37R2DbAdyLlqFqiEyKL6VVT+gzzKenY3MpodS7C4NpWthSxkK8HlOGxzWklb8T5GhI+ZB3xb3Yp08Jej8lGQf7xPldxpjlmdOpnKdFQqmuQBgeLEljN3cHlfzd9KHSBrF1g0VgBsX6cRUE2Kx8rmOFpsVn7VDH9u4vDHb3poS1V19NXOxoLsvHh1O6PJsxLgwNwtPOdXxVoIiXqXL81cOjI7UoMoisq3IwcsvWIV7a2Ngbg0nxP/D0+TjyTpyP7aWt4esL6AyKZiMr0snTCqW8HGzLwJ7fDH+4Gama60NlUs497uzJ44o/DP6XBozhn/H2l5+pTBKDNCoRiInszpw7kLXxGunmEH7azHmWljTaIId80HVaN/fWZgGY/xZRE6JNqeZNfX2uNSY4KG4C/FqEcc9hBxfuDCdi9cNQk+k7vE9MmNHNwwx2s= ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:16:34.812 DEBUG:teuthology.orchestra.run.smithi161:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx1sQLTEmyAgXsE9t6c87JfjiJetYiuI+ur7emKZG6fw1dcawIFy68YejeEc52YSsc93Ez/ySp37R2DbAdyLlqFqiEyKL6VVT+gzzKenY3MpodS7C4NpWthSxkK8HlOGxzWklb8T5GhI+ZB3xb3Yp08Jej8lGQf7xPldxpjlmdOpnKdFQqmuQBgeLEljN3cHlfzd9KHSBrF1g0VgBsX6cRUE2Kx8rmOFpsVn7VDH9u4vDHb3poS1V19NXOxoLsvHh1O6PJsxLgwNwtPOdXxVoIiXqXL81cOjI7UoMoisq3IwcsvWIV7a2Ngbg0nxP/D0+TjyTpyP7aWt4esL6AyKZiMr0snTCqW8HGzLwJ7fDH+4Gama60NlUs497uzJ44o/DP6XBozhn/H2l5+pTBKDNCoRiInszpw7kLXxGunmEH7azHmWljTaIId80HVaN/fWZgGY/xZRE6JNqeZNfX2uNSY4KG4C/FqEcc9hBxfuDCdi9cNQk+k7vE9MmNHNwwx2s= ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-03-27T10:16:34.859 INFO:teuthology.orchestra.run.smithi161.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx1sQLTEmyAgXsE9t6c87JfjiJetYiuI+ur7emKZG6fw1dcawIFy68YejeEc52YSsc93Ez/ySp37R2DbAdyLlqFqiEyKL6VVT+gzzKenY3MpodS7C4NpWthSxkK8HlOGxzWklb8T5GhI+ZB3xb3Yp08Jej8lGQf7xPldxpjlmdOpnKdFQqmuQBgeLEljN3cHlfzd9KHSBrF1g0VgBsX6cRUE2Kx8rmOFpsVn7VDH9u4vDHb3poS1V19NXOxoLsvHh1O6PJsxLgwNwtPOdXxVoIiXqXL81cOjI7UoMoisq3IwcsvWIV7a2Ngbg0nxP/D0+TjyTpyP7aWt4esL6AyKZiMr0snTCqW8HGzLwJ7fDH+4Gama60NlUs497uzJ44o/DP6XBozhn/H2l5+pTBKDNCoRiInszpw7kLXxGunmEH7azHmWljTaIId80HVaN/fWZgGY/xZRE6JNqeZNfX2uNSY4KG4C/FqEcc9hBxfuDCdi9cNQk+k7vE9MmNHNwwx2s= ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:16:34.871 DEBUG:teuthology.orchestra.run.smithi171:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx1sQLTEmyAgXsE9t6c87JfjiJetYiuI+ur7emKZG6fw1dcawIFy68YejeEc52YSsc93Ez/ySp37R2DbAdyLlqFqiEyKL6VVT+gzzKenY3MpodS7C4NpWthSxkK8HlOGxzWklb8T5GhI+ZB3xb3Yp08Jej8lGQf7xPldxpjlmdOpnKdFQqmuQBgeLEljN3cHlfzd9KHSBrF1g0VgBsX6cRUE2Kx8rmOFpsVn7VDH9u4vDHb3poS1V19NXOxoLsvHh1O6PJsxLgwNwtPOdXxVoIiXqXL81cOjI7UoMoisq3IwcsvWIV7a2Ngbg0nxP/D0+TjyTpyP7aWt4esL6AyKZiMr0snTCqW8HGzLwJ7fDH+4Gama60NlUs497uzJ44o/DP6XBozhn/H2l5+pTBKDNCoRiInszpw7kLXxGunmEH7azHmWljTaIId80HVaN/fWZgGY/xZRE6JNqeZNfX2uNSY4KG4C/FqEcc9hBxfuDCdi9cNQk+k7vE9MmNHNwwx2s= ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-03-27T10:16:34.916 INFO:teuthology.orchestra.run.smithi171.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCx1sQLTEmyAgXsE9t6c87JfjiJetYiuI+ur7emKZG6fw1dcawIFy68YejeEc52YSsc93Ez/ySp37R2DbAdyLlqFqiEyKL6VVT+gzzKenY3MpodS7C4NpWthSxkK8HlOGxzWklb8T5GhI+ZB3xb3Yp08Jej8lGQf7xPldxpjlmdOpnKdFQqmuQBgeLEljN3cHlfzd9KHSBrF1g0VgBsX6cRUE2Kx8rmOFpsVn7VDH9u4vDHb3poS1V19NXOxoLsvHh1O6PJsxLgwNwtPOdXxVoIiXqXL81cOjI7UoMoisq3IwcsvWIV7a2Ngbg0nxP/D0+TjyTpyP7aWt4esL6AyKZiMr0snTCqW8HGzLwJ7fDH+4Gama60NlUs497uzJ44o/DP6XBozhn/H2l5+pTBKDNCoRiInszpw7kLXxGunmEH7azHmWljTaIId80HVaN/fWZgGY/xZRE6JNqeZNfX2uNSY4KG4C/FqEcc9hBxfuDCdi9cNQk+k7vE9MmNHNwwx2s= ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:16:34.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-03-27T10:16:35.415 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:35 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/1770194536' entity='client.admin' 2023-03-27T10:16:35.801 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:16:37.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:37.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:37.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi159", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:16:37.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:37.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:16:37.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:37.763 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.763+0000 7f80a27e6700 1 -- 172.21.15.159:0/109192213 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f809c0ffc80 msgr2=0x7f809c100090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:37.764 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.763+0000 7f80a27e6700 1 --2- 172.21.15.159:0/109192213 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f809c0ffc80 0x7f809c100090 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f808c009910 tx=0x7f808c009c20 comp rx=0 tx=0).stop 2023-03-27T10:16:37.764 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.763+0000 7f80a27e6700 1 -- 172.21.15.159:0/109192213 shutdown_connections 2023-03-27T10:16:37.764 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.763+0000 7f80a27e6700 1 --2- 172.21.15.159:0/109192213 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f809c0ffc80 0x7f809c100090 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:37.764 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.763+0000 7f80a27e6700 1 -- 172.21.15.159:0/109192213 >> 172.21.15.159:0/109192213 conn(0x7f809c0f94c0 msgr2=0x7f809c0fb8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:37.764 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.763+0000 7f80a27e6700 1 -- 172.21.15.159:0/109192213 shutdown_connections 2023-03-27T10:16:37.764 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.763+0000 7f80a27e6700 1 -- 172.21.15.159:0/109192213 wait complete. 2023-03-27T10:16:37.764 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.764+0000 7f80a27e6700 1 Processor -- start 2023-03-27T10:16:37.765 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.764+0000 7f80a27e6700 1 -- start start 2023-03-27T10:16:37.766 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.764+0000 7f80a27e6700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f809c0ffc80 0x7f809c0fe3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:37.766 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.764+0000 7f80a27e6700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f809c197600 con 0x7f809c0ffc80 2023-03-27T10:16:37.766 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.764+0000 7f809bfff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f809c0ffc80 0x7f809c0fe3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:37.766 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.764+0000 7f809bfff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f809c0ffc80 0x7f809c0fe3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44666/0 (socket says 172.21.15.159:44666) 2023-03-27T10:16:37.766 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.764+0000 7f809bfff700 1 -- 172.21.15.159:0/992533676 learned_addr learned my addr 172.21.15.159:0/992533676 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:37.767 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.765+0000 7f809bfff700 1 -- 172.21.15.159:0/992533676 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f808c0095f0 con 0x7f809c0ffc80 2023-03-27T10:16:37.767 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.765+0000 7f809bfff700 1 --2- 172.21.15.159:0/992533676 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f809c0ffc80 0x7f809c0fe3d0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f808c006010 tx=0x7f808c0057c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:37.767 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.765+0000 7f80997fa700 1 -- 172.21.15.159:0/992533676 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f808c01d070 con 0x7f809c0ffc80 2023-03-27T10:16:37.767 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.765+0000 7f80997fa700 1 -- 172.21.15.159:0/992533676 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f808c0044e0 con 0x7f809c0ffc80 2023-03-27T10:16:37.767 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.765+0000 7f80a27e6700 1 -- 172.21.15.159:0/992533676 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f809c104510 con 0x7f809c0ffc80 2023-03-27T10:16:37.767 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.765+0000 7f80997fa700 1 -- 172.21.15.159:0/992533676 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f808c0039b0 con 0x7f809c0ffc80 2023-03-27T10:16:37.767 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.765+0000 7f80a27e6700 1 -- 172.21.15.159:0/992533676 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f809c104380 con 0x7f809c0ffc80 2023-03-27T10:16:37.768 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.766+0000 7f80997fa700 1 -- 172.21.15.159:0/992533676 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 13) v1 ==== 43566+0+0 (secure 0 0 0) 0x7f808c00c040 con 0x7f809c0ffc80 2023-03-27T10:16:37.768 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.766+0000 7f80997fa700 1 --2- 172.21.15.159:0/992533676 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f8084036910 0x7f8084038dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:37.768 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.766+0000 7f80997fa700 1 -- 172.21.15.159:0/992533676 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f808c049dd0 con 0x7f809c0ffc80 2023-03-27T10:16:37.768 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.766+0000 7f809b7fe700 1 --2- 172.21.15.159:0/992533676 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f8084036910 0x7f8084038dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:37.768 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.766+0000 7f80a27e6700 1 -- 172.21.15.159:0/992533676 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f809c18d240 con 0x7f809c0ffc80 2023-03-27T10:16:37.768 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.766+0000 7f809b7fe700 1 --2- 172.21.15.159:0/992533676 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f8084036910 0x7f8084038dc0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f8090006fd0 tx=0x7f8090006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:37.770 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.771+0000 7f80997fa700 1 -- 172.21.15.159:0/992533676 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f808c02a770 con 0x7f809c0ffc80 2023-03-27T10:16:37.951 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.951+0000 7f80a27e6700 1 -- 172.21.15.159:0/992533676 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/allow_ptrace}] v 0) v1 -- 0x7f809c02cdb0 con 0x7f809c0ffc80 2023-03-27T10:16:37.954 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.955+0000 7f80997fa700 1 -- 172.21.15.159:0/992533676 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/allow_ptrace}]=0 v9)=0 v9) v1 ==== 125+0+0 (secure 0 0 0) 0x7f808c013250 con 0x7f809c0ffc80 2023-03-27T10:16:37.956 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.957+0000 7f80a27e6700 1 -- 172.21.15.159:0/992533676 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f8084036910 msgr2=0x7f8084038dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:37.957 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.957+0000 7f80a27e6700 1 --2- 172.21.15.159:0/992533676 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f8084036910 0x7f8084038dc0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f8090006fd0 tx=0x7f8090006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:37.957 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.957+0000 7f80a27e6700 1 -- 172.21.15.159:0/992533676 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f809c0ffc80 msgr2=0x7f809c0fe3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:37.957 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.957+0000 7f80a27e6700 1 --2- 172.21.15.159:0/992533676 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f809c0ffc80 0x7f809c0fe3d0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f808c006010 tx=0x7f808c0057c0 comp rx=0 tx=0).stop 2023-03-27T10:16:37.957 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.957+0000 7f80a27e6700 1 -- 172.21.15.159:0/992533676 shutdown_connections 2023-03-27T10:16:37.957 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.957+0000 7f80a27e6700 1 --2- 172.21.15.159:0/992533676 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f8084036910 0x7f8084038dc0 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:37.957 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.957+0000 7f80a27e6700 1 --2- 172.21.15.159:0/992533676 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f809c0ffc80 0x7f809c0fe3d0 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:37.958 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.957+0000 7f80a27e6700 1 -- 172.21.15.159:0/992533676 >> 172.21.15.159:0/992533676 conn(0x7f809c0f94c0 msgr2=0x7f809c0fa120 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:37.958 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.957+0000 7f80a27e6700 1 -- 172.21.15.159:0/992533676 shutdown_connections 2023-03-27T10:16:37.958 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:37.957+0000 7f80a27e6700 1 -- 172.21.15.159:0/992533676 wait complete. 2023-03-27T10:16:38.329 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-03-27T10:16:38.330 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-03-27T10:16:38.969 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:16:39.207 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:38 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/992533676' entity='client.admin' 2023-03-27T10:16:39.208 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:38 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:16:39.208 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:38 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:40.168 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.167+0000 7f73be2a7700 1 -- 172.21.15.159:0/3264751548 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f73b80ff7f0 msgr2=0x7f73b80ffc00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:40.168 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.167+0000 7f73be2a7700 1 --2- 172.21.15.159:0/3264751548 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f73b80ff7f0 0x7f73b80ffc00 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f73a8009910 tx=0x7f73a8009c20 comp rx=0 tx=0).stop 2023-03-27T10:16:40.169 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.167+0000 7f73be2a7700 1 -- 172.21.15.159:0/3264751548 shutdown_connections 2023-03-27T10:16:40.169 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.167+0000 7f73be2a7700 1 --2- 172.21.15.159:0/3264751548 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f73b80ff7f0 0x7f73b80ffc00 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:40.169 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.167+0000 7f73be2a7700 1 -- 172.21.15.159:0/3264751548 >> 172.21.15.159:0/3264751548 conn(0x7f73b80fad80 msgr2=0x7f73b80fd1d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:40.169 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.167+0000 7f73be2a7700 1 -- 172.21.15.159:0/3264751548 shutdown_connections 2023-03-27T10:16:40.169 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.167+0000 7f73be2a7700 1 -- 172.21.15.159:0/3264751548 wait complete. 2023-03-27T10:16:40.169 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.168+0000 7f73be2a7700 1 Processor -- start 2023-03-27T10:16:40.169 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.168+0000 7f73be2a7700 1 -- start start 2023-03-27T10:16:40.170 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.168+0000 7f73be2a7700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f73b80ff7f0 0x7f73b81948b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:40.170 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.168+0000 7f73be2a7700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f73b8194df0 con 0x7f73b80ff7f0 2023-03-27T10:16:40.170 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.168+0000 7f73b7fff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f73b80ff7f0 0x7f73b81948b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:40.170 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.168+0000 7f73b7fff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f73b80ff7f0 0x7f73b81948b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44672/0 (socket says 172.21.15.159:44672) 2023-03-27T10:16:40.170 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.168+0000 7f73b7fff700 1 -- 172.21.15.159:0/3821751147 learned_addr learned my addr 172.21.15.159:0/3821751147 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:40.170 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.168+0000 7f73b7fff700 1 -- 172.21.15.159:0/3821751147 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f73a80095f0 con 0x7f73b80ff7f0 2023-03-27T10:16:40.170 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.169+0000 7f73b7fff700 1 --2- 172.21.15.159:0/3821751147 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f73b80ff7f0 0x7f73b81948b0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f73a8004f40 tx=0x7f73a80057c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:40.170 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.169+0000 7f73b57fa700 1 -- 172.21.15.159:0/3821751147 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f73a801d070 con 0x7f73b80ff7f0 2023-03-27T10:16:40.171 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.169+0000 7f73be2a7700 1 -- 172.21.15.159:0/3821751147 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f73b81056c0 con 0x7f73b80ff7f0 2023-03-27T10:16:40.171 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.169+0000 7f73b57fa700 1 -- 172.21.15.159:0/3821751147 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f73a800bc10 con 0x7f73b80ff7f0 2023-03-27T10:16:40.171 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.169+0000 7f73b57fa700 1 -- 172.21.15.159:0/3821751147 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f73a8003a70 con 0x7f73b80ff7f0 2023-03-27T10:16:40.171 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.169+0000 7f73be2a7700 1 -- 172.21.15.159:0/3821751147 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f73b8105530 con 0x7f73b80ff7f0 2023-03-27T10:16:40.172 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.170+0000 7f73b57fa700 1 -- 172.21.15.159:0/3821751147 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 13) v1 ==== 43566+0+0 (secure 0 0 0) 0x7f73a8028030 con 0x7f73b80ff7f0 2023-03-27T10:16:40.172 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.170+0000 7f73be2a7700 1 -- 172.21.15.159:0/3821751147 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f73b8105850 con 0x7f73b80ff7f0 2023-03-27T10:16:40.172 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.170+0000 7f73b57fa700 1 --2- 172.21.15.159:0/3821751147 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f73a00368c0 0x7f73a0038d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:40.172 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.170+0000 7f73b57fa700 1 -- 172.21.15.159:0/3821751147 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f73a8049990 con 0x7f73b80ff7f0 2023-03-27T10:16:40.172 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.170+0000 7f73b77fe700 1 --2- 172.21.15.159:0/3821751147 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f73a00368c0 0x7f73a0038d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:40.172 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.170+0000 7f73b77fe700 1 --2- 172.21.15.159:0/3821751147 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f73a00368c0 0x7f73a0038d70 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f73ac006fd0 tx=0x7f73ac006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:40.174 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.175+0000 7f73b57fa700 1 -- 172.21.15.159:0/3821751147 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f73a8029360 con 0x7f73b80ff7f0 2023-03-27T10:16:40.359 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.359+0000 7f73be2a7700 1 -- 172.21.15.159:0/3821751147 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}) v1 -- 0x7f73b8060c10 con 0x7f73a00368c0 2023-03-27T10:16:40.361 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.362+0000 7f73b57fa700 1 -- 172.21.15.159:0/3821751147 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7f73b8060c10 con 0x7f73a00368c0 2023-03-27T10:16:40.363 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.364+0000 7f73be2a7700 1 -- 172.21.15.159:0/3821751147 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f73a00368c0 msgr2=0x7f73a0038d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:40.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.364+0000 7f73be2a7700 1 --2- 172.21.15.159:0/3821751147 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f73a00368c0 0x7f73a0038d70 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f73ac006fd0 tx=0x7f73ac006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:40.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.364+0000 7f73be2a7700 1 -- 172.21.15.159:0/3821751147 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f73b80ff7f0 msgr2=0x7f73b81948b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:40.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.364+0000 7f73be2a7700 1 --2- 172.21.15.159:0/3821751147 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f73b80ff7f0 0x7f73b81948b0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f73a8004f40 tx=0x7f73a80057c0 comp rx=0 tx=0).stop 2023-03-27T10:16:40.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.364+0000 7f73be2a7700 1 -- 172.21.15.159:0/3821751147 shutdown_connections 2023-03-27T10:16:40.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.364+0000 7f73be2a7700 1 --2- 172.21.15.159:0/3821751147 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f73a00368c0 0x7f73a0038d70 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:40.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.364+0000 7f73be2a7700 1 --2- 172.21.15.159:0/3821751147 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f73b80ff7f0 0x7f73b81948b0 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:40.365 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.364+0000 7f73be2a7700 1 -- 172.21.15.159:0/3821751147 >> 172.21.15.159:0/3821751147 conn(0x7f73b80fad80 msgr2=0x7f73b80fba50 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:40.365 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.364+0000 7f73be2a7700 1 -- 172.21.15.159:0/3821751147 shutdown_connections 2023-03-27T10:16:40.365 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:40.364+0000 7f73be2a7700 1 -- 172.21.15.159:0/3821751147 wait complete. 2023-03-27T10:16:40.828 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi161 2023-03-27T10:16:40.829 DEBUG:teuthology.orchestra.run.smithi161:> set -ex 2023-03-27T10:16:40.829 DEBUG:teuthology.orchestra.run.smithi161:> dd of=/etc/ceph/ceph.conf 2023-03-27T10:16:40.849 DEBUG:teuthology.orchestra.run.smithi161:> set -ex 2023-03-27T10:16:40.850 DEBUG:teuthology.orchestra.run.smithi161:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-03-27T10:16:40.909 INFO:tasks.cephadm:Adding host smithi161 to orchestrator... 2023-03-27T10:16:40.909 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch host add smithi161 2023-03-27T10:16:41.567 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:16:41.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:41 smithi159 ceph-mon[93039]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:16:41.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:41 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:41.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:41 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:16:41.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:41 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:16:41.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:41 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:16:41.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:41 smithi159 ceph-mon[93039]: Updating smithi159:/etc/ceph/ceph.conf 2023-03-27T10:16:41.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:41 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:41.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:41 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:41.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:41 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:42.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:42 smithi159 ceph-mon[93039]: Updating smithi159:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.conf 2023-03-27T10:16:42.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:42 smithi159 ceph-mon[93039]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2023-03-27T10:16:42.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:42 smithi159 ceph-mon[93039]: Updating smithi159:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.client.admin.keyring 2023-03-27T10:16:42.836 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.837+0000 7f9f388c0700 1 -- 172.21.15.159:0/1557198954 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9f34105800 msgr2=0x7f9f34105bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:42.836 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.837+0000 7f9f388c0700 1 --2- 172.21.15.159:0/1557198954 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9f34105800 0x7f9f34105bd0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f9f28009910 tx=0x7f9f28009c20 comp rx=0 tx=0).stop 2023-03-27T10:16:42.836 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.837+0000 7f9f388c0700 1 -- 172.21.15.159:0/1557198954 shutdown_connections 2023-03-27T10:16:42.836 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.837+0000 7f9f388c0700 1 --2- 172.21.15.159:0/1557198954 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9f34105800 0x7f9f34105bd0 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:42.837 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.837+0000 7f9f388c0700 1 -- 172.21.15.159:0/1557198954 >> 172.21.15.159:0/1557198954 conn(0x7f9f340fb340 msgr2=0x7f9f340fd750 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:42.837 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.837+0000 7f9f388c0700 1 -- 172.21.15.159:0/1557198954 shutdown_connections 2023-03-27T10:16:42.837 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.837+0000 7f9f388c0700 1 -- 172.21.15.159:0/1557198954 wait complete. 2023-03-27T10:16:42.839 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.838+0000 7f9f388c0700 1 Processor -- start 2023-03-27T10:16:42.839 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.838+0000 7f9f388c0700 1 -- start start 2023-03-27T10:16:42.839 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.838+0000 7f9f388c0700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9f34105800 0x7f9f3419a9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:42.839 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.838+0000 7f9f388c0700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f3419aef0 con 0x7f9f34105800 2023-03-27T10:16:42.839 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.838+0000 7f9f3259c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9f34105800 0x7f9f3419a9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:42.839 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.838+0000 7f9f3259c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9f34105800 0x7f9f3419a9b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44678/0 (socket says 172.21.15.159:44678) 2023-03-27T10:16:42.840 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.838+0000 7f9f3259c700 1 -- 172.21.15.159:0/1199349917 learned_addr learned my addr 172.21.15.159:0/1199349917 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:42.840 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.838+0000 7f9f3259c700 1 -- 172.21.15.159:0/1199349917 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f280095f0 con 0x7f9f34105800 2023-03-27T10:16:42.840 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.838+0000 7f9f3259c700 1 --2- 172.21.15.159:0/1199349917 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9f34105800 0x7f9f3419a9b0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f9f28004f40 tx=0x7f9f28005020 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:42.840 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.839+0000 7f9f237fe700 1 -- 172.21.15.159:0/1199349917 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f9f2801d070 con 0x7f9f34105800 2023-03-27T10:16:42.840 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.839+0000 7f9f388c0700 1 -- 172.21.15.159:0/1199349917 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9f34109c60 con 0x7f9f34105800 2023-03-27T10:16:42.840 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.839+0000 7f9f237fe700 1 -- 172.21.15.159:0/1199349917 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9f2800be20 con 0x7f9f34105800 2023-03-27T10:16:42.841 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.839+0000 7f9f237fe700 1 -- 172.21.15.159:0/1199349917 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f9f28022d00 con 0x7f9f34105800 2023-03-27T10:16:42.841 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.839+0000 7f9f388c0700 1 -- 172.21.15.159:0/1199349917 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9f34109ad0 con 0x7f9f34105800 2023-03-27T10:16:42.841 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.839+0000 7f9f237fe700 1 -- 172.21.15.159:0/1199349917 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 13) v1 ==== 43566+0+0 (secure 0 0 0) 0x7f9f28029030 con 0x7f9f34105800 2023-03-27T10:16:42.841 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.840+0000 7f9f237fe700 1 --2- 172.21.15.159:0/1199349917 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f9f1c0368f0 0x7f9f1c038da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:42.842 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.840+0000 7f9f237fe700 1 -- 172.21.15.159:0/1199349917 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f9f28049d30 con 0x7f9f34105800 2023-03-27T10:16:42.842 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.840+0000 7f9f388c0700 1 -- 172.21.15.159:0/1199349917 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9f34109df0 con 0x7f9f34105800 2023-03-27T10:16:42.842 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.840+0000 7f9f31d9b700 1 --2- 172.21.15.159:0/1199349917 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f9f1c0368f0 0x7f9f1c038da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:42.842 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.840+0000 7f9f31d9b700 1 --2- 172.21.15.159:0/1199349917 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f9f1c0368f0 0x7f9f1c038da0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f9f24006fd0 tx=0x7f9f24006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:42.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:42.844+0000 7f9f237fe700 1 -- 172.21.15.159:0/1199349917 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f9f28027030 con 0x7f9f34105800 2023-03-27T10:16:43.031 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:43.033+0000 7f9f388c0700 1 -- 172.21.15.159:0/1199349917 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi161", "target": ["mon-mgr", ""]}) v1 -- 0x7f9f3419b460 con 0x7f9f1c0368f0 2023-03-27T10:16:43.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:43 smithi159 ceph-mon[93039]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi161", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:16:44.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:44 smithi159 ceph-mon[93039]: Deploying cephadm binary to smithi161 2023-03-27T10:16:44.918 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:44.919+0000 7f9f237fe700 1 -- 172.21.15.159:0/1199349917 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+49 (secure 0 0 0) 0x7f9f3419b460 con 0x7f9f1c0368f0 2023-03-27T10:16:44.918 INFO:teuthology.orchestra.run.smithi159.stdout:Added host 'smithi161' with addr '172.21.15.161' 2023-03-27T10:16:44.920 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:44.921+0000 7f9f388c0700 1 -- 172.21.15.159:0/1199349917 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f9f1c0368f0 msgr2=0x7f9f1c038da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:44.920 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:44.921+0000 7f9f388c0700 1 --2- 172.21.15.159:0/1199349917 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f9f1c0368f0 0x7f9f1c038da0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f9f24006fd0 tx=0x7f9f24006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:44.920 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:44.921+0000 7f9f388c0700 1 -- 172.21.15.159:0/1199349917 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9f34105800 msgr2=0x7f9f3419a9b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:44.920 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:44.921+0000 7f9f388c0700 1 --2- 172.21.15.159:0/1199349917 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9f34105800 0x7f9f3419a9b0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f9f28004f40 tx=0x7f9f28005020 comp rx=0 tx=0).stop 2023-03-27T10:16:44.920 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:44.921+0000 7f9f388c0700 1 -- 172.21.15.159:0/1199349917 shutdown_connections 2023-03-27T10:16:44.920 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:44.921+0000 7f9f388c0700 1 --2- 172.21.15.159:0/1199349917 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f9f1c0368f0 0x7f9f1c038da0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:44.921 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:44.921+0000 7f9f388c0700 1 --2- 172.21.15.159:0/1199349917 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9f34105800 0x7f9f3419a9b0 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:44.921 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:44.921+0000 7f9f388c0700 1 -- 172.21.15.159:0/1199349917 >> 172.21.15.159:0/1199349917 conn(0x7f9f340fb340 msgr2=0x7f9f340fbf80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:44.921 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:44.921+0000 7f9f388c0700 1 -- 172.21.15.159:0/1199349917 shutdown_connections 2023-03-27T10:16:44.921 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:44.921+0000 7f9f388c0700 1 -- 172.21.15.159:0/1199349917 wait complete. 2023-03-27T10:16:45.330 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch host ls --format=json 2023-03-27T10:16:45.975 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:16:46.174 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:45 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:46.175 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:45 smithi159 ceph-mon[93039]: Added host smithi161 2023-03-27T10:16:46.175 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:45 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:16:46.175 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:45 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:47.164 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.164+0000 7f89755c2700 1 -- 172.21.15.159:0/1345789241 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f89701047b0 msgr2=0x7f8970104b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:47.165 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.164+0000 7f89755c2700 1 --2- 172.21.15.159:0/1345789241 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f89701047b0 0x7f8970104b80 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f896000b0a0 tx=0x7f896000b3b0 comp rx=0 tx=0).stop 2023-03-27T10:16:47.165 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.164+0000 7f89755c2700 1 -- 172.21.15.159:0/1345789241 shutdown_connections 2023-03-27T10:16:47.165 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.164+0000 7f89755c2700 1 --2- 172.21.15.159:0/1345789241 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f89701047b0 0x7f8970104b80 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:47.165 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.164+0000 7f89755c2700 1 -- 172.21.15.159:0/1345789241 >> 172.21.15.159:0/1345789241 conn(0x7f8970075910 msgr2=0x7f8970075d10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:47.165 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.164+0000 7f89755c2700 1 -- 172.21.15.159:0/1345789241 shutdown_connections 2023-03-27T10:16:47.165 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.164+0000 7f89755c2700 1 -- 172.21.15.159:0/1345789241 wait complete. 2023-03-27T10:16:47.166 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.165+0000 7f89755c2700 1 Processor -- start 2023-03-27T10:16:47.166 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.165+0000 7f89755c2700 1 -- start start 2023-03-27T10:16:47.166 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.165+0000 7f89755c2700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f89701047b0 0x7f897019a8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:47.166 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.165+0000 7f89755c2700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f897019ae10 con 0x7f89701047b0 2023-03-27T10:16:47.166 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.166+0000 7f896effd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f89701047b0 0x7f897019a8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:47.166 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.166+0000 7f896effd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f89701047b0 0x7f897019a8d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44704/0 (socket says 172.21.15.159:44704) 2023-03-27T10:16:47.167 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.166+0000 7f896effd700 1 -- 172.21.15.159:0/3213410781 learned_addr learned my addr 172.21.15.159:0/3213410781 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:47.167 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.166+0000 7f896effd700 1 -- 172.21.15.159:0/3213410781 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8960009d00 con 0x7f89701047b0 2023-03-27T10:16:47.167 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.166+0000 7f896effd700 1 --2- 172.21.15.159:0/3213410781 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f89701047b0 0x7f897019a8d0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f89600093e0 tx=0x7f896000bd40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:47.167 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.166+0000 7f8967fff700 1 -- 172.21.15.159:0/3213410781 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f896000e040 con 0x7f89701047b0 2023-03-27T10:16:47.167 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.166+0000 7f8967fff700 1 -- 172.21.15.159:0/3213410781 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8960003d40 con 0x7f89701047b0 2023-03-27T10:16:47.167 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.166+0000 7f89755c2700 1 -- 172.21.15.159:0/3213410781 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8970109b60 con 0x7f89701047b0 2023-03-27T10:16:47.167 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.166+0000 7f8967fff700 1 -- 172.21.15.159:0/3213410781 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f896001d5e0 con 0x7f89701047b0 2023-03-27T10:16:47.167 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.166+0000 7f89755c2700 1 -- 172.21.15.159:0/3213410781 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f89701099d0 con 0x7f89701047b0 2023-03-27T10:16:47.168 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.167+0000 7f8967fff700 1 -- 172.21.15.159:0/3213410781 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 13) v1 ==== 43566+0+0 (secure 0 0 0) 0x7f896001d8a0 con 0x7f89701047b0 2023-03-27T10:16:47.168 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.167+0000 7f89755c2700 1 -- 172.21.15.159:0/3213410781 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8970109cf0 con 0x7f89701047b0 2023-03-27T10:16:47.168 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.167+0000 7f8967fff700 1 --2- 172.21.15.159:0/3213410781 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f895c036890 0x7f895c038d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:47.169 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.167+0000 7f8967fff700 1 -- 172.21.15.159:0/3213410781 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f896004a180 con 0x7f89701047b0 2023-03-27T10:16:47.169 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.167+0000 7f896e7fc700 1 --2- 172.21.15.159:0/3213410781 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f895c036890 0x7f895c038d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:47.169 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.168+0000 7f896e7fc700 1 --2- 172.21.15.159:0/3213410781 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f895c036890 0x7f895c038d40 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f8958006fd0 tx=0x7f8958006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:47.170 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.172+0000 7f8967fff700 1 -- 172.21.15.159:0/3213410781 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f896001a380 con 0x7f89701047b0 2023-03-27T10:16:47.356 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.358+0000 7f89755c2700 1 -- 172.21.15.159:0/3213410781 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f89700008d0 con 0x7f895c036890 2023-03-27T10:16:47.357 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.359+0000 7f8967fff700 1 -- 172.21.15.159:0/3213410781 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+161 (secure 0 0 0) 0x7f89700008d0 con 0x7f895c036890 2023-03-27T10:16:47.358 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:16:47.358 INFO:teuthology.orchestra.run.smithi159.stdout:[{"addr": "172.21.15.159", "hostname": "smithi159", "labels": [], "status": ""}, {"addr": "172.21.15.161", "hostname": "smithi161", "labels": [], "status": ""}] 2023-03-27T10:16:47.360 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.361+0000 7f89755c2700 1 -- 172.21.15.159:0/3213410781 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f895c036890 msgr2=0x7f895c038d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:47.360 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.361+0000 7f89755c2700 1 --2- 172.21.15.159:0/3213410781 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f895c036890 0x7f895c038d40 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f8958006fd0 tx=0x7f8958006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:47.360 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.361+0000 7f89755c2700 1 -- 172.21.15.159:0/3213410781 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f89701047b0 msgr2=0x7f897019a8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:47.360 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.361+0000 7f89755c2700 1 --2- 172.21.15.159:0/3213410781 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f89701047b0 0x7f897019a8d0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f89600093e0 tx=0x7f896000bd40 comp rx=0 tx=0).stop 2023-03-27T10:16:47.360 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.361+0000 7f89755c2700 1 -- 172.21.15.159:0/3213410781 shutdown_connections 2023-03-27T10:16:47.360 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.361+0000 7f89755c2700 1 --2- 172.21.15.159:0/3213410781 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f895c036890 0x7f895c038d40 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:47.361 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.361+0000 7f89755c2700 1 --2- 172.21.15.159:0/3213410781 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f89701047b0 0x7f897019a8d0 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:47.361 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.361+0000 7f89755c2700 1 -- 172.21.15.159:0/3213410781 >> 172.21.15.159:0/3213410781 conn(0x7f8970075910 msgr2=0x7f8970103c80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:47.361 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.362+0000 7f89755c2700 1 -- 172.21.15.159:0/3213410781 shutdown_connections 2023-03-27T10:16:47.361 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:47.362+0000 7f89755c2700 1 -- 172.21.15.159:0/3213410781 wait complete. 2023-03-27T10:16:47.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:47 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:47.729 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi171 2023-03-27T10:16:47.730 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2023-03-27T10:16:47.730 DEBUG:teuthology.orchestra.run.smithi171:> dd of=/etc/ceph/ceph.conf 2023-03-27T10:16:47.750 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2023-03-27T10:16:47.750 DEBUG:teuthology.orchestra.run.smithi171:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-03-27T10:16:47.809 INFO:tasks.cephadm:Adding host smithi171 to orchestrator... 2023-03-27T10:16:47.809 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch host add smithi171 2023-03-27T10:16:48.447 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:16:48.640 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:48 smithi159 ceph-mon[93039]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:16:48.641 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:48 smithi159 ceph-mon[93039]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-03-27T10:16:48.641 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:48 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:49.681 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.681+0000 7f617ea7b700 1 -- 172.21.15.159:0/2244778975 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6178103660 msgr2=0x7f6178103a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:49.681 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.681+0000 7f617ea7b700 1 --2- 172.21.15.159:0/2244778975 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6178103660 0x7f6178103a30 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f616c009910 tx=0x7f616c009c20 comp rx=0 tx=0).stop 2023-03-27T10:16:49.681 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.681+0000 7f617ea7b700 1 -- 172.21.15.159:0/2244778975 shutdown_connections 2023-03-27T10:16:49.682 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.681+0000 7f617ea7b700 1 --2- 172.21.15.159:0/2244778975 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6178103660 0x7f6178103a30 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:49.682 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.681+0000 7f617ea7b700 1 -- 172.21.15.159:0/2244778975 >> 172.21.15.159:0/2244778975 conn(0x7f6178075910 msgr2=0x7f6178075d10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:49.682 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.681+0000 7f617ea7b700 1 -- 172.21.15.159:0/2244778975 shutdown_connections 2023-03-27T10:16:49.682 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.681+0000 7f617ea7b700 1 -- 172.21.15.159:0/2244778975 wait complete. 2023-03-27T10:16:49.682 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.682+0000 7f617ea7b700 1 Processor -- start 2023-03-27T10:16:49.682 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.682+0000 7f617ea7b700 1 -- start start 2023-03-27T10:16:49.682 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.682+0000 7f617ea7b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6178103660 0x7f61781998e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:49.683 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.682+0000 7f617ea7b700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6178199e20 con 0x7f6178103660 2023-03-27T10:16:49.684 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.682+0000 7f617c817700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6178103660 0x7f61781998e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:49.684 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.682+0000 7f617c817700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6178103660 0x7f61781998e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44710/0 (socket says 172.21.15.159:44710) 2023-03-27T10:16:49.684 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.682+0000 7f617c817700 1 -- 172.21.15.159:0/178320445 learned_addr learned my addr 172.21.15.159:0/178320445 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:49.684 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.682+0000 7f617c817700 1 -- 172.21.15.159:0/178320445 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f616c0095f0 con 0x7f6178103660 2023-03-27T10:16:49.684 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.682+0000 7f617c817700 1 --2- 172.21.15.159:0/178320445 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6178103660 0x7f61781998e0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f616c0092e0 tx=0x7f616c004ff0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:49.684 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.682+0000 7f6175ffb700 1 -- 172.21.15.159:0/178320445 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f616c01d070 con 0x7f6178103660 2023-03-27T10:16:49.685 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.683+0000 7f617ea7b700 1 -- 172.21.15.159:0/178320445 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6178108a70 con 0x7f6178103660 2023-03-27T10:16:49.685 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.683+0000 7f6175ffb700 1 -- 172.21.15.159:0/178320445 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f616c00bc10 con 0x7f6178103660 2023-03-27T10:16:49.685 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.683+0000 7f6175ffb700 1 -- 172.21.15.159:0/178320445 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f616c003930 con 0x7f6178103660 2023-03-27T10:16:49.685 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.683+0000 7f617ea7b700 1 -- 172.21.15.159:0/178320445 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f61781088e0 con 0x7f6178103660 2023-03-27T10:16:49.685 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.684+0000 7f6175ffb700 1 -- 172.21.15.159:0/178320445 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 13) v1 ==== 43566+0+0 (secure 0 0 0) 0x7f616c028030 con 0x7f6178103660 2023-03-27T10:16:49.685 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.684+0000 7f617ea7b700 1 -- 172.21.15.159:0/178320445 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6178108c00 con 0x7f6178103660 2023-03-27T10:16:49.685 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.684+0000 7f6175ffb700 1 --2- 172.21.15.159:0/178320445 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f61680368a0 0x7f6168038d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:49.686 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.684+0000 7f6175ffb700 1 -- 172.21.15.159:0/178320445 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f616c049870 con 0x7f6178103660 2023-03-27T10:16:49.686 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.684+0000 7f6177fff700 1 --2- 172.21.15.159:0/178320445 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f61680368a0 0x7f6168038d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:49.686 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.684+0000 7f6177fff700 1 --2- 172.21.15.159:0/178320445 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f61680368a0 0x7f6168038d50 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f6164006fd0 tx=0x7f6164006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:49.688 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.689+0000 7f6175ffb700 1 -- 172.21.15.159:0/178320445 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f616c02ad80 con 0x7f6178103660 2023-03-27T10:16:49.875 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:49.875+0000 7f617ea7b700 1 -- 172.21.15.159:0/178320445 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi171", "target": ["mon-mgr", ""]}) v1 -- 0x7f6178194ee0 con 0x7f61680368a0 2023-03-27T10:16:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:50 smithi159 ceph-mon[93039]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:16:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:50 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:50 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:50 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:50 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:50 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi161", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:16:50.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:50 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:16:50.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:50 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:16:51.766 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:51.766+0000 7f6175ffb700 1 -- 172.21.15.159:0/178320445 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+49 (secure 0 0 0) 0x7f6178194ee0 con 0x7f61680368a0 2023-03-27T10:16:51.767 INFO:teuthology.orchestra.run.smithi159.stdout:Added host 'smithi171' with addr '172.21.15.171' 2023-03-27T10:16:51.768 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:51.769+0000 7f617ea7b700 1 -- 172.21.15.159:0/178320445 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f61680368a0 msgr2=0x7f6168038d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:51.769 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:51.769+0000 7f617ea7b700 1 --2- 172.21.15.159:0/178320445 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f61680368a0 0x7f6168038d50 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f6164006fd0 tx=0x7f6164006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:51.769 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:51.769+0000 7f617ea7b700 1 -- 172.21.15.159:0/178320445 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6178103660 msgr2=0x7f61781998e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:51.770 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:51.769+0000 7f617ea7b700 1 --2- 172.21.15.159:0/178320445 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6178103660 0x7f61781998e0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f616c0092e0 tx=0x7f616c004ff0 comp rx=0 tx=0).stop 2023-03-27T10:16:51.770 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:51.769+0000 7f617ea7b700 1 -- 172.21.15.159:0/178320445 shutdown_connections 2023-03-27T10:16:51.770 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:51.769+0000 7f617ea7b700 1 --2- 172.21.15.159:0/178320445 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f61680368a0 0x7f6168038d50 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:51.770 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:51.769+0000 7f617ea7b700 1 --2- 172.21.15.159:0/178320445 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6178103660 0x7f61781998e0 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:51.771 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:51.769+0000 7f617ea7b700 1 -- 172.21.15.159:0/178320445 >> 172.21.15.159:0/178320445 conn(0x7f6178075910 msgr2=0x7f6178102b10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:51.771 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:51.769+0000 7f617ea7b700 1 -- 172.21.15.159:0/178320445 shutdown_connections 2023-03-27T10:16:51.771 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:51.769+0000 7f617ea7b700 1 -- 172.21.15.159:0/178320445 wait complete. 2023-03-27T10:16:51.795 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:51 smithi159 ceph-mon[93039]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi171", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:16:51.795 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:51 smithi159 ceph-mon[93039]: Updating smithi161:/etc/ceph/ceph.conf 2023-03-27T10:16:51.795 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:51 smithi159 ceph-mon[93039]: Updating smithi161:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.conf 2023-03-27T10:16:51.795 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:51 smithi159 ceph-mon[93039]: Deploying cephadm binary to smithi171 2023-03-27T10:16:51.795 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:51 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:51.796 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:51 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:51.796 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:51 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:52.203 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch host ls --format=json 2023-03-27T10:16:52.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:52 smithi159 ceph-mon[93039]: Updating smithi161:/etc/ceph/ceph.client.admin.keyring 2023-03-27T10:16:52.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:52 smithi159 ceph-mon[93039]: Updating smithi161:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.client.admin.keyring 2023-03-27T10:16:52.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:52 smithi159 ceph-mon[93039]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:16:52.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:52 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:52.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:52 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:16:52.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:52 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:52.842 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:16:53.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:53 smithi159 ceph-mon[93039]: Added host smithi171 2023-03-27T10:16:53.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:53 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:54.046 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.046+0000 7fe94e50f700 1 -- 172.21.15.159:0/3269190204 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe948103660 msgr2=0x7fe948103a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:54.046 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.046+0000 7fe94e50f700 1 --2- 172.21.15.159:0/3269190204 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe948103660 0x7fe948103a30 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7fe93c009910 tx=0x7fe93c009c20 comp rx=0 tx=0).stop 2023-03-27T10:16:54.047 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.046+0000 7fe94e50f700 1 -- 172.21.15.159:0/3269190204 shutdown_connections 2023-03-27T10:16:54.047 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.046+0000 7fe94e50f700 1 --2- 172.21.15.159:0/3269190204 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe948103660 0x7fe948103a30 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:54.047 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.046+0000 7fe94e50f700 1 -- 172.21.15.159:0/3269190204 >> 172.21.15.159:0/3269190204 conn(0x7fe948075910 msgr2=0x7fe948075d10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:54.047 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.046+0000 7fe94e50f700 1 -- 172.21.15.159:0/3269190204 shutdown_connections 2023-03-27T10:16:54.047 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.046+0000 7fe94e50f700 1 -- 172.21.15.159:0/3269190204 wait complete. 2023-03-27T10:16:54.047 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.046+0000 7fe94e50f700 1 Processor -- start 2023-03-27T10:16:54.048 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.046+0000 7fe94e50f700 1 -- start start 2023-03-27T10:16:54.048 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.047+0000 7fe94e50f700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe948103660 0x7fe948100bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:54.048 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.047+0000 7fe94e50f700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe948102b50 con 0x7fe948103660 2023-03-27T10:16:54.048 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.047+0000 7fe947fff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe948103660 0x7fe948100bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:54.048 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.047+0000 7fe947fff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe948103660 0x7fe948100bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44736/0 (socket says 172.21.15.159:44736) 2023-03-27T10:16:54.048 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.047+0000 7fe947fff700 1 -- 172.21.15.159:0/2359242415 learned_addr learned my addr 172.21.15.159:0/2359242415 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:54.050 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.047+0000 7fe947fff700 1 -- 172.21.15.159:0/2359242415 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe93c0095f0 con 0x7fe948103660 2023-03-27T10:16:54.050 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.047+0000 7fe947fff700 1 --2- 172.21.15.159:0/2359242415 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe948103660 0x7fe948100bd0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fe93c004f40 tx=0x7fe93c005020 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:54.051 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.047+0000 7fe9457fa700 1 -- 172.21.15.159:0/2359242415 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fe93c01d070 con 0x7fe948103660 2023-03-27T10:16:54.051 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.047+0000 7fe9457fa700 1 -- 172.21.15.159:0/2359242415 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe93c00be20 con 0x7fe948103660 2023-03-27T10:16:54.051 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.047+0000 7fe9457fa700 1 -- 172.21.15.159:0/2359242415 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fe93c022d00 con 0x7fe948103660 2023-03-27T10:16:54.051 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.047+0000 7fe94e50f700 1 -- 172.21.15.159:0/2359242415 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe948108a70 con 0x7fe948103660 2023-03-27T10:16:54.051 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.047+0000 7fe94e50f700 1 -- 172.21.15.159:0/2359242415 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe9481088e0 con 0x7fe948103660 2023-03-27T10:16:54.052 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.048+0000 7fe9457fa700 1 -- 172.21.15.159:0/2359242415 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 13) v1 ==== 43566+0+0 (secure 0 0 0) 0x7fe93c029030 con 0x7fe948103660 2023-03-27T10:16:54.052 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.048+0000 7fe9457fa700 1 --2- 172.21.15.159:0/2359242415 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe9300368f0 0x7fe930038da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:54.052 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.048+0000 7fe9457fa700 1 -- 172.21.15.159:0/2359242415 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fe93c049df0 con 0x7fe948103660 2023-03-27T10:16:54.052 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.048+0000 7fe94e50f700 1 -- 172.21.15.159:0/2359242415 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe948108c00 con 0x7fe948103660 2023-03-27T10:16:54.052 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.048+0000 7fe9477fe700 1 --2- 172.21.15.159:0/2359242415 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe9300368f0 0x7fe930038da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:54.052 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.049+0000 7fe9477fe700 1 --2- 172.21.15.159:0/2359242415 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe9300368f0 0x7fe930038da0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7fe938006fd0 tx=0x7fe938006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:54.053 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.053+0000 7fe9457fa700 1 -- 172.21.15.159:0/2359242415 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fe93c027030 con 0x7fe948103660 2023-03-27T10:16:54.253 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.253+0000 7fe94e50f700 1 -- 172.21.15.159:0/2359242415 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fe948102210 con 0x7fe9300368f0 2023-03-27T10:16:54.254 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.254+0000 7fe9457fa700 1 -- 172.21.15.159:0/2359242415 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+241 (secure 0 0 0) 0x7fe948102210 con 0x7fe9300368f0 2023-03-27T10:16:54.254 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:16:54.254 INFO:teuthology.orchestra.run.smithi159.stdout:[{"addr": "172.21.15.159", "hostname": "smithi159", "labels": [], "status": ""}, {"addr": "172.21.15.161", "hostname": "smithi161", "labels": [], "status": ""}, {"addr": "172.21.15.171", "hostname": "smithi171", "labels": [], "status": ""}] 2023-03-27T10:16:54.256 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.256+0000 7fe94e50f700 1 -- 172.21.15.159:0/2359242415 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe9300368f0 msgr2=0x7fe930038da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:54.256 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.256+0000 7fe94e50f700 1 --2- 172.21.15.159:0/2359242415 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe9300368f0 0x7fe930038da0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7fe938006fd0 tx=0x7fe938006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:54.256 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.256+0000 7fe94e50f700 1 -- 172.21.15.159:0/2359242415 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe948103660 msgr2=0x7fe948100bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:54.256 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.256+0000 7fe94e50f700 1 --2- 172.21.15.159:0/2359242415 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe948103660 0x7fe948100bd0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fe93c004f40 tx=0x7fe93c005020 comp rx=0 tx=0).stop 2023-03-27T10:16:54.257 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.257+0000 7fe94e50f700 1 -- 172.21.15.159:0/2359242415 shutdown_connections 2023-03-27T10:16:54.257 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.257+0000 7fe94e50f700 1 --2- 172.21.15.159:0/2359242415 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe9300368f0 0x7fe930038da0 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:54.257 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.257+0000 7fe94e50f700 1 --2- 172.21.15.159:0/2359242415 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe948103660 0x7fe948100bd0 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:54.257 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.257+0000 7fe94e50f700 1 -- 172.21.15.159:0/2359242415 >> 172.21.15.159:0/2359242415 conn(0x7fe948075910 msgr2=0x7fe9480677f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:54.257 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.257+0000 7fe94e50f700 1 -- 172.21.15.159:0/2359242415 shutdown_connections 2023-03-27T10:16:54.257 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:54.257+0000 7fe94e50f700 1 -- 172.21.15.159:0/2359242415 wait complete. 2023-03-27T10:16:54.596 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:54 smithi159 ceph-mon[93039]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:16:54.596 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:54 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:54.597 INFO:tasks.cephadm:Setting crush tunables to default 2023-03-27T10:16:54.598 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph osd crush tunables default 2023-03-27T10:16:55.238 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:16:55.742 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:55 smithi159 ceph-mon[93039]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-03-27T10:16:56.447 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.446+0000 7f6499523700 1 -- 172.21.15.159:0/369679767 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6494103660 msgr2=0x7f6494103a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:56.447 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.446+0000 7f6499523700 1 --2- 172.21.15.159:0/369679767 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6494103660 0x7f6494103a30 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f6488009910 tx=0x7f6488009c20 comp rx=0 tx=0).stop 2023-03-27T10:16:56.447 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.446+0000 7f6499523700 1 -- 172.21.15.159:0/369679767 shutdown_connections 2023-03-27T10:16:56.447 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.446+0000 7f6499523700 1 --2- 172.21.15.159:0/369679767 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6494103660 0x7f6494103a30 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:56.447 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.446+0000 7f6499523700 1 -- 172.21.15.159:0/369679767 >> 172.21.15.159:0/369679767 conn(0x7f6494075910 msgr2=0x7f6494075d10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:56.448 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.446+0000 7f6499523700 1 -- 172.21.15.159:0/369679767 shutdown_connections 2023-03-27T10:16:56.448 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.446+0000 7f6499523700 1 -- 172.21.15.159:0/369679767 wait complete. 2023-03-27T10:16:56.448 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.446+0000 7f6499523700 1 Processor -- start 2023-03-27T10:16:56.448 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.447+0000 7f6499523700 1 -- start start 2023-03-27T10:16:56.448 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.447+0000 7f6499523700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6494103660 0x7f6494195a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:56.448 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.447+0000 7f6499523700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6494195fa0 con 0x7f6494103660 2023-03-27T10:16:56.449 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.447+0000 7f6492ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6494103660 0x7f6494195a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:56.449 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.447+0000 7f6492ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6494103660 0x7f6494195a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44742/0 (socket says 172.21.15.159:44742) 2023-03-27T10:16:56.449 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.447+0000 7f6492ffd700 1 -- 172.21.15.159:0/2402905318 learned_addr learned my addr 172.21.15.159:0/2402905318 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:16:56.449 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.447+0000 7f6492ffd700 1 -- 172.21.15.159:0/2402905318 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f64880095f0 con 0x7f6494103660 2023-03-27T10:16:56.449 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.447+0000 7f6492ffd700 1 --2- 172.21.15.159:0/2402905318 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6494103660 0x7f6494195a60 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f64880092e0 tx=0x7f6488004ff0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:56.449 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.447+0000 7f647bfff700 1 -- 172.21.15.159:0/2402905318 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f648801d070 con 0x7f6494103660 2023-03-27T10:16:56.449 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.447+0000 7f647bfff700 1 -- 172.21.15.159:0/2402905318 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f648800bc10 con 0x7f6494103660 2023-03-27T10:16:56.450 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.447+0000 7f6499523700 1 -- 172.21.15.159:0/2402905318 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6494108a70 con 0x7f6494103660 2023-03-27T10:16:56.450 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.447+0000 7f647bfff700 1 -- 172.21.15.159:0/2402905318 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f6488003930 con 0x7f6494103660 2023-03-27T10:16:56.451 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.447+0000 7f6499523700 1 -- 172.21.15.159:0/2402905318 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f64941088e0 con 0x7f6494103660 2023-03-27T10:16:56.451 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.448+0000 7f647bfff700 1 -- 172.21.15.159:0/2402905318 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 13) v1 ==== 43566+0+0 (secure 0 0 0) 0x7f6488028030 con 0x7f6494103660 2023-03-27T10:16:56.451 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.448+0000 7f6499523700 1 -- 172.21.15.159:0/2402905318 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6494108c00 con 0x7f6494103660 2023-03-27T10:16:56.451 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.448+0000 7f647bfff700 1 --2- 172.21.15.159:0/2402905318 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f647c0368a0 0x7f647c038d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:56.451 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.448+0000 7f647bfff700 1 -- 172.21.15.159:0/2402905318 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f64880498c0 con 0x7f6494103660 2023-03-27T10:16:56.451 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.449+0000 7f64927fc700 1 --2- 172.21.15.159:0/2402905318 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f647c0368a0 0x7f647c038d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:56.452 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.449+0000 7f64927fc700 1 --2- 172.21.15.159:0/2402905318 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f647c0368a0 0x7f647c038d50 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f6484006fd0 tx=0x7f6484006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:56.452 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.453+0000 7f647bfff700 1 -- 172.21.15.159:0/2402905318 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f6488029360 con 0x7f6494103660 2023-03-27T10:16:56.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:56 smithi159 ceph-mon[93039]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:16:56.641 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:56.642+0000 7f6499523700 1 -- 172.21.15.159:0/2402905318 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd crush tunables", "profile": "default"} v 0) v1 -- 0x7f6494067e00 con 0x7f6494103660 2023-03-27T10:16:57.413 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:57.414+0000 7f647bfff700 1 -- 172.21.15.159:0/2402905318 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd crush tunables", "profile": "default"}]=0 adjusted tunables profile to default v4) v1 ==== 124+0+0 (secure 0 0 0) 0x7f6488026070 con 0x7f6494103660 2023-03-27T10:16:57.415 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:57.416+0000 7f6499523700 1 -- 172.21.15.159:0/2402905318 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f647c0368a0 msgr2=0x7f647c038d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:57.415 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:57.416+0000 7f6499523700 1 --2- 172.21.15.159:0/2402905318 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f647c0368a0 0x7f647c038d50 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f6484006fd0 tx=0x7f6484006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:57.415 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:57.416+0000 7f6499523700 1 -- 172.21.15.159:0/2402905318 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6494103660 msgr2=0x7f6494195a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:57.416 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:57.416+0000 7f6499523700 1 --2- 172.21.15.159:0/2402905318 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6494103660 0x7f6494195a60 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f64880092e0 tx=0x7f6488004ff0 comp rx=0 tx=0).stop 2023-03-27T10:16:57.416 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:57.416+0000 7f6499523700 1 -- 172.21.15.159:0/2402905318 shutdown_connections 2023-03-27T10:16:57.416 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:57.416+0000 7f6499523700 1 --2- 172.21.15.159:0/2402905318 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f647c0368a0 0x7f647c038d50 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:57.416 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:57.416+0000 7f6499523700 1 --2- 172.21.15.159:0/2402905318 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6494103660 0x7f6494195a60 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:57.416 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:57.416+0000 7f6499523700 1 -- 172.21.15.159:0/2402905318 >> 172.21.15.159:0/2402905318 conn(0x7f6494075910 msgr2=0x7f6494102b10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:57.416 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:57.416+0000 7f6499523700 1 -- 172.21.15.159:0/2402905318 shutdown_connections 2023-03-27T10:16:57.416 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:16:57.416+0000 7f6499523700 1 -- 172.21.15.159:0/2402905318 wait complete. 2023-03-27T10:16:57.417 INFO:teuthology.orchestra.run.smithi159.stderr:adjusted tunables profile to default 2023-03-27T10:16:57.781 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:57 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/2402905318' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-03-27T10:16:57.781 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:57 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:57.781 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:57 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:57.782 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:57 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:57.782 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:57 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:57.782 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:57 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi171", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:16:57.782 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:57 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:16:57.782 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:57 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:16:57.783 INFO:tasks.cephadm:Adding mon.a on smithi159 2023-03-27T10:16:57.783 INFO:tasks.cephadm:Adding mon.b on smithi161 2023-03-27T10:16:57.784 INFO:tasks.cephadm:Adding mon.c on smithi171 2023-03-27T10:16:57.784 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch apply mon '3;smithi159:172.21.15.159=a;smithi161:172.21.15.161=b;smithi171:172.21.15.171=c' 2023-03-27T10:16:58.447 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.conf 2023-03-27T10:16:58.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:58 smithi159 ceph-mon[93039]: Updating smithi171:/etc/ceph/ceph.conf 2023-03-27T10:16:58.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:58 smithi159 ceph-mon[93039]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:16:58.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:58 smithi159 ceph-mon[93039]: Updating smithi171:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.conf 2023-03-27T10:16:58.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:58 smithi159 ceph-mon[93039]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2023-03-27T10:16:58.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:58 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/2402905318' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-03-27T10:16:58.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:58 smithi159 ceph-mon[93039]: osdmap e4: 0 total, 0 up, 0 in 2023-03-27T10:16:58.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:58 smithi159 ceph-mon[93039]: Updating smithi171:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.client.admin.keyring 2023-03-27T10:16:58.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:58 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:58.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:58 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:58.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:16:58 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:16:59.655 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.656+0000 7fafe8a22700 1 -- 172.21.15.171:0/454078198 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fafe4103660 msgr2=0x7fafe4103a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:59.656 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.656+0000 7fafe8a22700 1 --2- 172.21.15.171:0/454078198 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fafe4103660 0x7fafe4103a30 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7fafd8009910 tx=0x7fafd8009c20 comp rx=0 tx=0).stop 2023-03-27T10:16:59.656 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.656+0000 7fafe8a22700 1 -- 172.21.15.171:0/454078198 shutdown_connections 2023-03-27T10:16:59.656 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.656+0000 7fafe8a22700 1 --2- 172.21.15.171:0/454078198 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fafe4103660 0x7fafe4103a30 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:59.656 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.656+0000 7fafe8a22700 1 -- 172.21.15.171:0/454078198 >> 172.21.15.171:0/454078198 conn(0x7fafe4075910 msgr2=0x7fafe4075d10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:59.656 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.656+0000 7fafe8a22700 1 -- 172.21.15.171:0/454078198 shutdown_connections 2023-03-27T10:16:59.656 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.656+0000 7fafe8a22700 1 -- 172.21.15.171:0/454078198 wait complete. 2023-03-27T10:16:59.657 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.657+0000 7fafe8a22700 1 Processor -- start 2023-03-27T10:16:59.657 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.657+0000 7fafe8a22700 1 -- start start 2023-03-27T10:16:59.657 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.657+0000 7fafe8a22700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fafe4103660 0x7fafe419a8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:59.657 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.657+0000 7fafe8a22700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fafe419ae10 con 0x7fafe4103660 2023-03-27T10:16:59.658 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.657+0000 7fafe259c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fafe4103660 0x7fafe419a8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:59.659 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.657+0000 7fafe259c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fafe4103660 0x7fafe419a8d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.171:45394/0 (socket says 172.21.15.171:45394) 2023-03-27T10:16:59.659 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.657+0000 7fafe259c700 1 -- 172.21.15.171:0/1255824269 learned_addr learned my addr 172.21.15.171:0/1255824269 (peer_addr_for_me v2:172.21.15.171:0/0) 2023-03-27T10:16:59.659 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.657+0000 7fafe259c700 1 -- 172.21.15.171:0/1255824269 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fafd80095f0 con 0x7fafe4103660 2023-03-27T10:16:59.659 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.658+0000 7fafe259c700 1 --2- 172.21.15.171:0/1255824269 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fafe4103660 0x7fafe419a8d0 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fafd80092e0 tx=0x7fafd8004ff0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:59.659 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.658+0000 7fafd37fe700 1 -- 172.21.15.171:0/1255824269 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fafd801d070 con 0x7fafe4103660 2023-03-27T10:16:59.659 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.658+0000 7fafe8a22700 1 -- 172.21.15.171:0/1255824269 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fafe4108a70 con 0x7fafe4103660 2023-03-27T10:16:59.659 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.658+0000 7fafd37fe700 1 -- 172.21.15.171:0/1255824269 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fafd800bc10 con 0x7fafe4103660 2023-03-27T10:16:59.660 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.658+0000 7fafd37fe700 1 -- 172.21.15.171:0/1255824269 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fafd8003930 con 0x7fafe4103660 2023-03-27T10:16:59.660 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.658+0000 7fafe8a22700 1 -- 172.21.15.171:0/1255824269 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fafe41088e0 con 0x7fafe4103660 2023-03-27T10:16:59.660 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.659+0000 7fafd37fe700 1 -- 172.21.15.171:0/1255824269 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 13) v1 ==== 43566+0+0 (secure 0 0 0) 0x7fafd8028030 con 0x7fafe4103660 2023-03-27T10:16:59.660 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.659+0000 7fafe8a22700 1 -- 172.21.15.171:0/1255824269 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fafe4108c00 con 0x7fafe4103660 2023-03-27T10:16:59.660 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.659+0000 7fafd37fe700 1 --2- 172.21.15.171:0/1255824269 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fafcc0368a0 0x7fafcc038d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:16:59.660 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.659+0000 7fafd37fe700 1 -- 172.21.15.171:0/1255824269 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fafd8049870 con 0x7fafe4103660 2023-03-27T10:16:59.660 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.659+0000 7fafe1d9b700 1 --2- 172.21.15.171:0/1255824269 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fafcc0368a0 0x7fafcc038d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:16:59.660 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.660+0000 7fafe1d9b700 1 --2- 172.21.15.171:0/1255824269 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fafcc0368a0 0x7fafcc038d50 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fafd4006fd0 tx=0x7fafd4006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:16:59.663 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.664+0000 7fafd37fe700 1 -- 172.21.15.171:0/1255824269 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fafd802ad80 con 0x7fafe4103660 2023-03-27T10:16:59.853 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.854+0000 7fafe8a22700 1 -- 172.21.15.171:0/1255824269 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi159:172.21.15.159=a;smithi161:172.21.15.161=b;smithi171:172.21.15.171=c", "target": ["mon-mgr", ""]}) v1 -- 0x7fafe4195ff0 con 0x7fafcc0368a0 2023-03-27T10:16:59.860 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.861+0000 7fafd37fe700 1 -- 172.21.15.171:0/1255824269 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7fafe4195ff0 con 0x7fafcc0368a0 2023-03-27T10:16:59.860 INFO:teuthology.orchestra.run.smithi171.stdout:Scheduled mon update... 2023-03-27T10:16:59.862 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.863+0000 7fafe8a22700 1 -- 172.21.15.171:0/1255824269 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fafcc0368a0 msgr2=0x7fafcc038d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:59.862 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.863+0000 7fafe8a22700 1 --2- 172.21.15.171:0/1255824269 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fafcc0368a0 0x7fafcc038d50 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fafd4006fd0 tx=0x7fafd4006e40 comp rx=0 tx=0).stop 2023-03-27T10:16:59.862 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.863+0000 7fafe8a22700 1 -- 172.21.15.171:0/1255824269 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fafe4103660 msgr2=0x7fafe419a8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:16:59.862 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.863+0000 7fafe8a22700 1 --2- 172.21.15.171:0/1255824269 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fafe4103660 0x7fafe419a8d0 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fafd80092e0 tx=0x7fafd8004ff0 comp rx=0 tx=0).stop 2023-03-27T10:16:59.862 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.863+0000 7fafe8a22700 1 -- 172.21.15.171:0/1255824269 shutdown_connections 2023-03-27T10:16:59.863 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.863+0000 7fafe8a22700 1 --2- 172.21.15.171:0/1255824269 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fafcc0368a0 0x7fafcc038d50 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:59.863 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.863+0000 7fafe8a22700 1 --2- 172.21.15.171:0/1255824269 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fafe4103660 0x7fafe419a8d0 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:16:59.863 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.863+0000 7fafe8a22700 1 -- 172.21.15.171:0/1255824269 >> 172.21.15.171:0/1255824269 conn(0x7fafe4075910 msgr2=0x7fafe40677f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:16:59.863 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.863+0000 7fafe8a22700 1 -- 172.21.15.171:0/1255824269 shutdown_connections 2023-03-27T10:16:59.863 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:16:59.863+0000 7fafe8a22700 1 -- 172.21.15.171:0/1255824269 wait complete. 2023-03-27T10:17:00.313 DEBUG:teuthology.orchestra.run.smithi161:mon.b> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mon.b.service 2023-03-27T10:17:00.315 DEBUG:teuthology.orchestra.run.smithi171:mon.c> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mon.c.service 2023-03-27T10:17:00.317 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-03-27T10:17:00.317 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph mon dump -f json 2023-03-27T10:17:00.346 INFO:journalctl@ceph.mon.c.smithi171.stdout:-- Logs begin at Mon 2023-03-27 10:03:15 UTC. -- 2023-03-27T10:17:00.351 INFO:journalctl@ceph.mon.b.smithi161.stdout:-- Logs begin at Mon 2023-03-27 10:02:59 UTC. -- 2023-03-27T10:17:00.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:00 smithi159 ceph-mon[93039]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:00.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:00.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:00.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:00.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:00.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:00.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-03-27T10:17:00.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:00.951 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.conf 2023-03-27T10:17:01.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:01 smithi159 ceph-mon[93039]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi159:172.21.15.159=a;smithi161:172.21.15.161=b;smithi171:172.21.15.171=c", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:17:01.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:01 smithi159 ceph-mon[93039]: Saving service mon spec with placement smithi159:172.21.15.159=a;smithi161:172.21.15.161=b;smithi171:172.21.15.171=c;count:3 2023-03-27T10:17:01.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:01 smithi159 ceph-mon[93039]: Deploying daemon mon.c on smithi171 2023-03-27T10:17:02.586 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.586+0000 7fd3620a5700 1 -- 172.21.15.171:0/596852152 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd35c105810 msgr2=0x7fd35c105be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:02.586 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.586+0000 7fd3620a5700 1 --2- 172.21.15.171:0/596852152 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd35c105810 0x7fd35c105be0 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fd34c009910 tx=0x7fd34c009c20 comp rx=0 tx=0).stop 2023-03-27T10:17:02.586 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.586+0000 7fd3620a5700 1 -- 172.21.15.171:0/596852152 shutdown_connections 2023-03-27T10:17:02.586 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.586+0000 7fd3620a5700 1 --2- 172.21.15.171:0/596852152 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd35c105810 0x7fd35c105be0 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:02.586 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.586+0000 7fd3620a5700 1 -- 172.21.15.171:0/596852152 >> 172.21.15.171:0/596852152 conn(0x7fd35c0fb330 msgr2=0x7fd35c0fd740 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:17:02.587 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.586+0000 7fd3620a5700 1 -- 172.21.15.171:0/596852152 shutdown_connections 2023-03-27T10:17:02.587 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.586+0000 7fd3620a5700 1 -- 172.21.15.171:0/596852152 wait complete. 2023-03-27T10:17:02.587 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.587+0000 7fd3620a5700 1 Processor -- start 2023-03-27T10:17:02.588 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.587+0000 7fd3620a5700 1 -- start start 2023-03-27T10:17:02.588 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.588+0000 7fd3620a5700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd35c105810 0x7fd35c19aa60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:02.588 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.588+0000 7fd3620a5700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd35c19afa0 con 0x7fd35c105810 2023-03-27T10:17:02.588 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.588+0000 7fd35b7fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd35c105810 0x7fd35c19aa60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:02.588 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.588+0000 7fd35b7fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd35c105810 0x7fd35c19aa60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.171:45400/0 (socket says 172.21.15.171:45400) 2023-03-27T10:17:02.589 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.588+0000 7fd35b7fe700 1 -- 172.21.15.171:0/222368296 learned_addr learned my addr 172.21.15.171:0/222368296 (peer_addr_for_me v2:172.21.15.171:0/0) 2023-03-27T10:17:02.589 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.588+0000 7fd35b7fe700 1 -- 172.21.15.171:0/222368296 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd34c0095f0 con 0x7fd35c105810 2023-03-27T10:17:02.589 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.588+0000 7fd35b7fe700 1 --2- 172.21.15.171:0/222368296 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd35c105810 0x7fd35c19aa60 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fd34c004f40 tx=0x7fd34c005020 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:17:02.589 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.589+0000 7fd358ff9700 1 -- 172.21.15.171:0/222368296 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd34c01d070 con 0x7fd35c105810 2023-03-27T10:17:02.589 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.589+0000 7fd3620a5700 1 -- 172.21.15.171:0/222368296 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd35c109c70 con 0x7fd35c105810 2023-03-27T10:17:02.589 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.589+0000 7fd358ff9700 1 -- 172.21.15.171:0/222368296 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd34c00be20 con 0x7fd35c105810 2023-03-27T10:17:02.589 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.589+0000 7fd358ff9700 1 -- 172.21.15.171:0/222368296 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd34c022d00 con 0x7fd35c105810 2023-03-27T10:17:02.590 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.589+0000 7fd3620a5700 1 -- 172.21.15.171:0/222368296 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd35c109ae0 con 0x7fd35c105810 2023-03-27T10:17:02.590 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.590+0000 7fd358ff9700 1 -- 172.21.15.171:0/222368296 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 13) v1 ==== 43566+0+0 (secure 0 0 0) 0x7fd34c029030 con 0x7fd35c105810 2023-03-27T10:17:02.590 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.590+0000 7fd3620a5700 1 -- 172.21.15.171:0/222368296 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd35c109e00 con 0x7fd35c105810 2023-03-27T10:17:02.591 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.590+0000 7fd358ff9700 1 --2- 172.21.15.171:0/222368296 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd3440368a0 0x7fd344038d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:02.591 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.590+0000 7fd358ff9700 1 -- 172.21.15.171:0/222368296 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fd34c049c90 con 0x7fd35c105810 2023-03-27T10:17:02.591 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.590+0000 7fd35affd700 1 --2- 172.21.15.171:0/222368296 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd3440368a0 0x7fd344038d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:02.591 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.590+0000 7fd35affd700 1 --2- 172.21.15.171:0/222368296 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd3440368a0 0x7fd344038d50 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fd350006fd0 tx=0x7fd350006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:17:02.594 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.595+0000 7fd358ff9700 1 -- 172.21.15.171:0/222368296 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fd34c02be20 con 0x7fd35c105810 2023-03-27T10:17:02.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:02 smithi159 ceph-mon[93039]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:02.884 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.884+0000 7fd3620a5700 1 -- 172.21.15.171:0/222368296 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fd35c061e00 con 0x7fd35c105810 2023-03-27T10:17:02.884 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:17:02.885 INFO:teuthology.orchestra.run.smithi171.stdout:{"epoch":1,"fsid":"2c3cd140-cc88-11ed-9afe-001a4aab830c","modified":"2023-03-27T10:15:30.484420Z","created":"2023-03-27T10:15:30.484420Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-03-27T10:17:02.885 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.885+0000 7fd358ff9700 1 -- 172.21.15.171:0/222368296 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+740 (secure 0 0 0) 0x7fd34c027030 con 0x7fd35c105810 2023-03-27T10:17:02.886 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.887+0000 7fd3620a5700 1 -- 172.21.15.171:0/222368296 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd3440368a0 msgr2=0x7fd344038d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:02.887 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.887+0000 7fd3620a5700 1 --2- 172.21.15.171:0/222368296 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd3440368a0 0x7fd344038d50 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fd350006fd0 tx=0x7fd350006e40 comp rx=0 tx=0).stop 2023-03-27T10:17:02.887 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.887+0000 7fd3620a5700 1 -- 172.21.15.171:0/222368296 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd35c105810 msgr2=0x7fd35c19aa60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:02.887 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.887+0000 7fd3620a5700 1 --2- 172.21.15.171:0/222368296 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd35c105810 0x7fd35c19aa60 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fd34c004f40 tx=0x7fd34c005020 comp rx=0 tx=0).stop 2023-03-27T10:17:02.887 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.887+0000 7fd3620a5700 1 -- 172.21.15.171:0/222368296 shutdown_connections 2023-03-27T10:17:02.887 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.887+0000 7fd3620a5700 1 --2- 172.21.15.171:0/222368296 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd3440368a0 0x7fd344038d50 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:02.887 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.887+0000 7fd3620a5700 1 --2- 172.21.15.171:0/222368296 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd35c105810 0x7fd35c19aa60 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:02.887 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.887+0000 7fd3620a5700 1 -- 172.21.15.171:0/222368296 >> 172.21.15.171:0/222368296 conn(0x7fd35c0fb330 msgr2=0x7fd35c0fd200 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:17:02.888 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.888+0000 7fd3620a5700 1 -- 172.21.15.171:0/222368296 shutdown_connections 2023-03-27T10:17:02.888 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:02.888+0000 7fd3620a5700 1 -- 172.21.15.171:0/222368296 wait complete. 2023-03-27T10:17:02.888 INFO:teuthology.orchestra.run.smithi171.stderr:dumped monmap epoch 1 2023-03-27T10:17:03.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 systemd[1]: Starting Ceph mon.c for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:17:03.654 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 podman[94768]: 2023-03-27T10:17:03.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:03 smithi159 ceph-mon[93039]: from='client.? 172.21.15.171:0/222368296' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-03-27T10:17:03.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: set uid:gid to 167:167 (ceph:ceph) 2023-03-27T10:17:03.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: ceph version 18.0.0-2533-gb70fdf97 (b70fdf973c2ffa23c0d8aa6918519da05d922302) reef (dev), process ceph-mon, pid 7 2023-03-27T10:17:03.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: pidfile_write: ignore empty --pid-file 2023-03-27T10:17:03.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 bash[94768]: ea04c5186c69c15cd34d1af89060a74a6e7fca13ce6a771d5437de431d3ebb55 2023-03-27T10:17:03.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: load: jerasure load: lrc 2023-03-27T10:17:03.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: RocksDB version: 6.15.5 2023-03-27T10:17:03.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-03-27T10:17:03.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Compile date Mar 16 2023 2023-03-27T10:17:03.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: DB SUMMARY 2023-03-27T10:17:03.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: DB Session ID: 3GW4NZ185J72S1KLI7L9 2023-03-27T10:17:03.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: CURRENT file: CURRENT 2023-03-27T10:17:03.979 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: IDENTITY file: IDENTITY 2023-03-27T10:17:03.979 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-03-27T10:17:03.979 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-03-27T10:17:03.979 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2023-03-27T10:17:03.979 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.error_if_exists: 0 2023-03-27T10:17:03.979 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.create_if_missing: 0 2023-03-27T10:17:03.979 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.paranoid_checks: 1 2023-03-27T10:17:03.980 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-03-27T10:17:03.980 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.env: 0x55efdfd1a540 2023-03-27T10:17:03.980 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.fs: Posix File System 2023-03-27T10:17:03.980 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.info_log: 0x55efe21ce3a0 2023-03-27T10:17:03.980 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_file_opening_threads: 16 2023-03-27T10:17:03.980 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.statistics: (nil) 2023-03-27T10:17:03.980 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.use_fsync: 0 2023-03-27T10:17:03.980 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_log_file_size: 0 2023-03-27T10:17:03.981 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-03-27T10:17:03.981 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.log_file_time_to_roll: 0 2023-03-27T10:17:03.981 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.keep_log_file_num: 1000 2023-03-27T10:17:03.981 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.recycle_log_file_num: 0 2023-03-27T10:17:03.981 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.allow_fallocate: 1 2023-03-27T10:17:03.981 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.allow_mmap_reads: 0 2023-03-27T10:17:03.981 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.allow_mmap_writes: 0 2023-03-27T10:17:03.981 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.use_direct_reads: 0 2023-03-27T10:17:03.982 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-03-27T10:17:03.982 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.create_missing_column_families: 0 2023-03-27T10:17:03.982 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.db_log_dir: 2023-03-27T10:17:03.982 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-03-27T10:17:03.982 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.table_cache_numshardbits: 6 2023-03-27T10:17:03.982 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.WAL_ttl_seconds: 0 2023-03-27T10:17:03.982 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.WAL_size_limit_MB: 0 2023-03-27T10:17:03.983 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-03-27T10:17:03.983 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-03-27T10:17:03.983 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.is_fd_close_on_exec: 1 2023-03-27T10:17:03.983 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.advise_random_on_open: 1 2023-03-27T10:17:03.983 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.db_write_buffer_size: 0 2023-03-27T10:17:03.983 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.write_buffer_manager: 0x55efe2456f60 2023-03-27T10:17:03.983 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-03-27T10:17:03.983 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-03-27T10:17:03.984 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-03-27T10:17:03.984 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.use_adaptive_mutex: 0 2023-03-27T10:17:03.984 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.rate_limiter: (nil) 2023-03-27T10:17:03.984 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-03-27T10:17:03.984 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.wal_recovery_mode: 2 2023-03-27T10:17:03.984 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.enable_thread_tracking: 0 2023-03-27T10:17:03.984 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.enable_pipelined_write: 0 2023-03-27T10:17:03.984 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.unordered_write: 0 2023-03-27T10:17:03.985 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-03-27T10:17:03.985 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-03-27T10:17:03.985 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-03-27T10:17:03.985 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-03-27T10:17:03.985 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.row_cache: None 2023-03-27T10:17:03.985 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.wal_filter: None 2023-03-27T10:17:03.985 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-03-27T10:17:03.986 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.allow_ingest_behind: 0 2023-03-27T10:17:03.986 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.preserve_deletes: 0 2023-03-27T10:17:03.986 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.two_write_queues: 0 2023-03-27T10:17:03.986 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.manual_wal_flush: 0 2023-03-27T10:17:03.986 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.atomic_flush: 0 2023-03-27T10:17:03.986 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-03-27T10:17:03.987 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.persist_stats_to_disk: 0 2023-03-27T10:17:03.987 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.write_dbid_to_manifest: 0 2023-03-27T10:17:03.987 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.log_readahead_size: 0 2023-03-27T10:17:03.987 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-03-27T10:17:03.988 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.best_efforts_recovery: 0 2023-03-27T10:17:03.988 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-03-27T10:17:03.988 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-03-27T10:17:03.988 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.allow_data_in_errors: 0 2023-03-27T10:17:03.989 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.db_host_id: __hostname__ 2023-03-27T10:17:03.989 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_background_jobs: 2 2023-03-27T10:17:03.989 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_background_compactions: -1 2023-03-27T10:17:03.990 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_subcompactions: 1 2023-03-27T10:17:03.990 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-03-27T10:17:03.990 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-03-27T10:17:03.990 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.delayed_write_rate : 16777216 2023-03-27T10:17:03.991 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_total_wal_size: 0 2023-03-27T10:17:03.991 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-03-27T10:17:03.991 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.stats_dump_period_sec: 600 2023-03-27T10:17:03.991 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.stats_persist_period_sec: 600 2023-03-27T10:17:03.992 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-03-27T10:17:03.992 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_open_files: -1 2023-03-27T10:17:03.992 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.bytes_per_sync: 0 2023-03-27T10:17:03.992 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.wal_bytes_per_sync: 0 2023-03-27T10:17:03.993 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.strict_bytes_per_sync: 0 2023-03-27T10:17:03.993 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compaction_readahead_size: 0 2023-03-27T10:17:03.993 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_background_flushes: -1 2023-03-27T10:17:03.994 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Compression algorithms supported: 2023-03-27T10:17:03.994 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-03-27T10:17:03.994 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: kZSTD supported: 0 2023-03-27T10:17:03.994 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: kXpressCompression supported: 0 2023-03-27T10:17:03.995 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: kLZ4HCCompression supported: 1 2023-03-27T10:17:03.995 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: kZlibCompression supported: 1 2023-03-27T10:17:03.995 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: kSnappyCompression supported: 1 2023-03-27T10:17:03.995 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: kLZ4Compression supported: 1 2023-03-27T10:17:03.996 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: kBZip2Compression supported: 0 2023-03-27T10:17:03.996 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Fast CRC32 supported: Supported on x86 2023-03-27T10:17:03.996 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2023-03-27T10:17:03.996 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-03-27T10:17:03.997 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-03-27T10:17:03.997 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.merge_operator: 2023-03-27T10:17:03.997 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compaction_filter: None 2023-03-27T10:17:03.997 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compaction_filter_factory: None 2023-03-27T10:17:03.998 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.sst_partitioner_factory: None 2023-03-27T10:17:03.998 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.memtable_factory: SkipListFactory 2023-03-27T10:17:03.998 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.table_factory: BlockBasedTable 2023-03-27T10:17:03.998 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55efe21de5f0) 2023-03-27T10:17:03.999 INFO:journalctl@ceph.mon.c.smithi171.stdout: cache_index_and_filter_blocks: 1 2023-03-27T10:17:03.999 INFO:journalctl@ceph.mon.c.smithi171.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-03-27T10:17:03.999 INFO:journalctl@ceph.mon.c.smithi171.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-03-27T10:17:03.999 INFO:journalctl@ceph.mon.c.smithi171.stdout: pin_top_level_index_and_filter: 1 2023-03-27T10:17:04.000 INFO:journalctl@ceph.mon.c.smithi171.stdout: index_type: 0 2023-03-27T10:17:04.000 INFO:journalctl@ceph.mon.c.smithi171.stdout: data_block_index_type: 0 2023-03-27T10:17:04.000 INFO:journalctl@ceph.mon.c.smithi171.stdout: index_shortening: 1 2023-03-27T10:17:04.000 INFO:journalctl@ceph.mon.c.smithi171.stdout: data_block_hash_table_util_ratio: 0.750000 2023-03-27T10:17:04.001 INFO:journalctl@ceph.mon.c.smithi171.stdout: hash_index_allow_collision: 1 2023-03-27T10:17:04.001 INFO:journalctl@ceph.mon.c.smithi171.stdout: checksum: 1 2023-03-27T10:17:04.001 INFO:journalctl@ceph.mon.c.smithi171.stdout: no_block_cache: 0 2023-03-27T10:17:04.002 INFO:journalctl@ceph.mon.c.smithi171.stdout: block_cache: 0x55efe221f350 2023-03-27T10:17:04.002 INFO:journalctl@ceph.mon.c.smithi171.stdout: block_cache_name: BinnedLRUCache 2023-03-27T10:17:04.002 INFO:journalctl@ceph.mon.c.smithi171.stdout: block_cache_options: 2023-03-27T10:17:04.002 INFO:journalctl@ceph.mon.c.smithi171.stdout: capacity : 536870912 2023-03-27T10:17:04.003 INFO:journalctl@ceph.mon.c.smithi171.stdout: num_shard_bits : 4 2023-03-27T10:17:04.003 INFO:journalctl@ceph.mon.c.smithi171.stdout: strict_capacity_limit : 0 2023-03-27T10:17:04.003 INFO:journalctl@ceph.mon.c.smithi171.stdout: high_pri_pool_ratio: 0.000 2023-03-27T10:17:04.003 INFO:journalctl@ceph.mon.c.smithi171.stdout: block_cache_compressed: (nil) 2023-03-27T10:17:04.004 INFO:journalctl@ceph.mon.c.smithi171.stdout: persistent_cache: (nil) 2023-03-27T10:17:04.004 INFO:journalctl@ceph.mon.c.smithi171.stdout: block_size: 4096 2023-03-27T10:17:04.004 INFO:journalctl@ceph.mon.c.smithi171.stdout: block_size_deviation: 10 2023-03-27T10:17:04.004 INFO:journalctl@ceph.mon.c.smithi171.stdout: block_restart_interval: 16 2023-03-27T10:17:04.005 INFO:journalctl@ceph.mon.c.smithi171.stdout: index_block_restart_interval: 1 2023-03-27T10:17:04.005 INFO:journalctl@ceph.mon.c.smithi171.stdout: metadata_block_size: 4096 2023-03-27T10:17:04.005 INFO:journalctl@ceph.mon.c.smithi171.stdout: partition_filters: 0 2023-03-27T10:17:04.005 INFO:journalctl@ceph.mon.c.smithi171.stdout: use_delta_encoding: 1 2023-03-27T10:17:04.006 INFO:journalctl@ceph.mon.c.smithi171.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-03-27T10:17:04.006 INFO:journalctl@ceph.mon.c.smithi171.stdout: whole_key_filtering: 1 2023-03-27T10:17:04.006 INFO:journalctl@ceph.mon.c.smithi171.stdout: verify_compression: 0 2023-03-27T10:17:04.006 INFO:journalctl@ceph.mon.c.smithi171.stdout: read_amp_bytes_per_bit: 0 2023-03-27T10:17:04.007 INFO:journalctl@ceph.mon.c.smithi171.stdout: format_version: 4 2023-03-27T10:17:04.007 INFO:journalctl@ceph.mon.c.smithi171.stdout: enable_index_compression: 1 2023-03-27T10:17:04.007 INFO:journalctl@ceph.mon.c.smithi171.stdout: block_align: 0 2023-03-27T10:17:04.007 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.write_buffer_size: 33554432 2023-03-27T10:17:04.008 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_write_buffer_number: 2 2023-03-27T10:17:04.008 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compression: NoCompression 2023-03-27T10:17:04.008 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.bottommost_compression: Disabled 2023-03-27T10:17:04.008 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.prefix_extractor: nullptr 2023-03-27T10:17:04.009 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-03-27T10:17:04.009 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.num_levels: 7 2023-03-27T10:17:04.009 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-03-27T10:17:04.009 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-03-27T10:17:04.010 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-03-27T10:17:04.010 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-03-27T10:17:04.010 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-03-27T10:17:04.010 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-03-27T10:17:04.011 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-03-27T10:17:04.011 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-03-27T10:17:04.011 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-03-27T10:17:04.011 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-03-27T10:17:04.012 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compression_opts.window_bits: -14 2023-03-27T10:17:04.012 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compression_opts.level: 32767 2023-03-27T10:17:04.014 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compression_opts.strategy: 0 2023-03-27T10:17:04.014 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-03-27T10:17:04.014 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-03-27T10:17:04.015 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-03-27T10:17:04.015 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compression_opts.enabled: false 2023-03-27T10:17:04.015 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-03-27T10:17:04.015 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-03-27T10:17:04.016 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-03-27T10:17:04.016 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.target_file_size_base: 67108864 2023-03-27T10:17:04.016 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.target_file_size_multiplier: 1 2023-03-27T10:17:04.016 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-03-27T10:17:04.017 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-03-27T10:17:04.017 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-03-27T10:17:04.017 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-03-27T10:17:04.017 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-03-27T10:17:04.018 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-03-27T10:17:04.018 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-03-27T10:17:04.018 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-03-27T10:17:04.018 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-03-27T10:17:04.019 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-03-27T10:17:04.019 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-03-27T10:17:04.019 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-03-27T10:17:04.019 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.arena_block_size: 4194304 2023-03-27T10:17:04.020 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-03-27T10:17:04.020 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-03-27T10:17:04.020 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-03-27T10:17:04.020 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.disable_auto_compactions: 0 2023-03-27T10:17:04.021 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-03-27T10:17:04.021 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-03-27T10:17:04.021 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-03-27T10:17:04.021 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-03-27T10:17:04.022 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-03-27T10:17:04.022 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-03-27T10:17:04.022 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-03-27T10:17:04.022 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-03-27T10:17:04.023 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-03-27T10:17:04.023 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-03-27T10:17:04.023 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.table_properties_collectors: 2023-03-27T10:17:04.023 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.inplace_update_support: 0 2023-03-27T10:17:04.024 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.inplace_update_num_locks: 10000 2023-03-27T10:17:04.024 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-03-27T10:17:04.024 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-03-27T10:17:04.024 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.memtable_huge_page_size: 0 2023-03-27T10:17:04.025 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.bloom_locality: 0 2023-03-27T10:17:04.025 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.max_successive_merges: 0 2023-03-27T10:17:04.025 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.optimize_filters_for_hits: 0 2023-03-27T10:17:04.026 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.paranoid_file_checks: 0 2023-03-27T10:17:04.026 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.force_consistency_checks: 1 2023-03-27T10:17:04.026 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.report_bg_io_stats: 0 2023-03-27T10:17:04.026 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.ttl: 2592000 2023-03-27T10:17:04.027 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.periodic_compaction_seconds: 0 2023-03-27T10:17:04.027 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.enable_blob_files: false 2023-03-27T10:17:04.027 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.min_blob_size: 0 2023-03-27T10:17:04.027 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.blob_file_size: 268435456 2023-03-27T10:17:04.028 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.blob_compression_type: NoCompression 2023-03-27T10:17:04.028 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.enable_blob_garbage_collection: false 2023-03-27T10:17:04.028 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-03-27T10:17:04.028 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-03-27T10:17:04.029 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-03-27T10:17:04.029 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-03-27T10:17:04.029 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: EVENT_LOG_v1 {"time_micros": 1679912223726865, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-03-27T10:17:04.029 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-03-27T10:17:04.030 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-03-27T10:17:04.030 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: EVENT_LOG_v1 {"time_micros": 1679912223727133, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1679912223, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "ed085c55-a2ff-4d2e-bfd3-69bb79df9f37", "db_session_id": "3GW4NZ185J72S1KLI7L9"}} 2023-03-27T10:17:04.030 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-03-27T10:17:04.030 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: EVENT_LOG_v1 {"time_micros": 1679912223727443, "job": 1, "event": "recovery_finished"} 2023-03-27T10:17:04.031 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-03-27T10:17:04.031 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55efe21b3c00 2023-03-27T10:17:04.031 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: DB pointer 0x55efe21d0000 2023-03-27T10:17:04.031 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-03-27T10:17:04.032 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-03-27T10:17:04.032 INFO:journalctl@ceph.mon.c.smithi171.stdout: ** DB Stats ** 2023-03-27T10:17:04.032 INFO:journalctl@ceph.mon.c.smithi171.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-03-27T10:17:04.032 INFO:journalctl@ceph.mon.c.smithi171.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-03-27T10:17:04.033 INFO:journalctl@ceph.mon.c.smithi171.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-03-27T10:17:04.033 INFO:journalctl@ceph.mon.c.smithi171.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-03-27T10:17:04.033 INFO:journalctl@ceph.mon.c.smithi171.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-03-27T10:17:04.033 INFO:journalctl@ceph.mon.c.smithi171.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-03-27T10:17:04.034 INFO:journalctl@ceph.mon.c.smithi171.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-03-27T10:17:04.034 INFO:journalctl@ceph.mon.c.smithi171.stdout: 2023-03-27T10:17:04.034 INFO:journalctl@ceph.mon.c.smithi171.stdout: ** Compaction Stats [default] ** 2023-03-27T10:17:04.034 INFO:journalctl@ceph.mon.c.smithi171.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-03-27T10:17:04.035 INFO:journalctl@ceph.mon.c.smithi171.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-03-27T10:17:04.035 INFO:journalctl@ceph.mon.c.smithi171.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-03-27T10:17:04.035 INFO:journalctl@ceph.mon.c.smithi171.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-03-27T10:17:04.035 INFO:journalctl@ceph.mon.c.smithi171.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-03-27T10:17:04.036 INFO:journalctl@ceph.mon.c.smithi171.stdout: 2023-03-27T10:17:04.036 INFO:journalctl@ceph.mon.c.smithi171.stdout: ** Compaction Stats [default] ** 2023-03-27T10:17:04.036 INFO:journalctl@ceph.mon.c.smithi171.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-03-27T10:17:04.036 INFO:journalctl@ceph.mon.c.smithi171.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-03-27T10:17:04.037 INFO:journalctl@ceph.mon.c.smithi171.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-03-27T10:17:04.037 INFO:journalctl@ceph.mon.c.smithi171.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-03-27T10:17:04.037 INFO:journalctl@ceph.mon.c.smithi171.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-03-27T10:17:04.038 INFO:journalctl@ceph.mon.c.smithi171.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-03-27T10:17:04.038 INFO:journalctl@ceph.mon.c.smithi171.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-03-27T10:17:04.038 INFO:journalctl@ceph.mon.c.smithi171.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-03-27T10:17:04.038 INFO:journalctl@ceph.mon.c.smithi171.stdout: AddFile(Keys): cumulative 0, interval 0 2023-03-27T10:17:04.039 INFO:journalctl@ceph.mon.c.smithi171.stdout: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-03-27T10:17:04.039 INFO:journalctl@ceph.mon.c.smithi171.stdout: Interval compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-03-27T10:17:04.039 INFO:journalctl@ceph.mon.c.smithi171.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-03-27T10:17:04.039 INFO:journalctl@ceph.mon.c.smithi171.stdout: 2023-03-27T10:17:04.040 INFO:journalctl@ceph.mon.c.smithi171.stdout: ** File Read Latency Histogram By Level [default] ** 2023-03-27T10:17:04.040 INFO:journalctl@ceph.mon.c.smithi171.stdout: 2023-03-27T10:17:04.040 INFO:journalctl@ceph.mon.c.smithi171.stdout: ** Compaction Stats [default] ** 2023-03-27T10:17:04.040 INFO:journalctl@ceph.mon.c.smithi171.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-03-27T10:17:04.041 INFO:journalctl@ceph.mon.c.smithi171.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-03-27T10:17:04.041 INFO:journalctl@ceph.mon.c.smithi171.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-03-27T10:17:04.041 INFO:journalctl@ceph.mon.c.smithi171.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-03-27T10:17:04.041 INFO:journalctl@ceph.mon.c.smithi171.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-03-27T10:17:04.042 INFO:journalctl@ceph.mon.c.smithi171.stdout: 2023-03-27T10:17:04.042 INFO:journalctl@ceph.mon.c.smithi171.stdout: ** Compaction Stats [default] ** 2023-03-27T10:17:04.042 INFO:journalctl@ceph.mon.c.smithi171.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-03-27T10:17:04.042 INFO:journalctl@ceph.mon.c.smithi171.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-03-27T10:17:04.043 INFO:journalctl@ceph.mon.c.smithi171.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-03-27T10:17:04.043 INFO:journalctl@ceph.mon.c.smithi171.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-03-27T10:17:04.043 INFO:journalctl@ceph.mon.c.smithi171.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-03-27T10:17:04.043 INFO:journalctl@ceph.mon.c.smithi171.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-03-27T10:17:04.044 INFO:journalctl@ceph.mon.c.smithi171.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-03-27T10:17:04.044 INFO:journalctl@ceph.mon.c.smithi171.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-03-27T10:17:04.044 INFO:journalctl@ceph.mon.c.smithi171.stdout: AddFile(Keys): cumulative 0, interval 0 2023-03-27T10:17:04.044 INFO:journalctl@ceph.mon.c.smithi171.stdout: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-03-27T10:17:04.045 INFO:journalctl@ceph.mon.c.smithi171.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-03-27T10:17:04.045 INFO:journalctl@ceph.mon.c.smithi171.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-03-27T10:17:04.045 INFO:journalctl@ceph.mon.c.smithi171.stdout: 2023-03-27T10:17:04.045 INFO:journalctl@ceph.mon.c.smithi171.stdout: ** File Read Latency Histogram By Level [default] ** 2023-03-27T10:17:04.046 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-03-27T10:17:04.046 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: using public_addr v2:172.21.15.171:0/0 -> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] 2023-03-27T10:17:04.046 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: starting mon.c rank -1 at public addrs [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] at bind addrs [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon_data /var/lib/ceph/mon/ceph-c fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:17:04.046 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: mon.c@-1(???) e0 preinit fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:17:04.047 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 systemd[1]: Started Ceph mon.c for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:17:04.047 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: mon.c@-1(synchronizing).mds e1 new map 2023-03-27T10:17:04.047 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: mon.c@-1(synchronizing).mds e1 print_map 2023-03-27T10:17:04.047 INFO:journalctl@ceph.mon.c.smithi171.stdout: e1 2023-03-27T10:17:04.047 INFO:journalctl@ceph.mon.c.smithi171.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-03-27T10:17:04.047 INFO:journalctl@ceph.mon.c.smithi171.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-03-27T10:17:04.047 INFO:journalctl@ceph.mon.c.smithi171.stdout: legacy client fscid: -1 2023-03-27T10:17:04.047 INFO:journalctl@ceph.mon.c.smithi171.stdout: 2023-03-27T10:17:04.048 INFO:journalctl@ceph.mon.c.smithi171.stdout: No filesystems configured 2023-03-27T10:17:04.048 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-03-27T10:17:04.048 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-03-27T10:17:04.048 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-03-27T10:17:04.048 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-03-27T10:17:04.048 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-03-27T10:17:04.048 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-03-27T10:17:04.049 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-03-27T10:17:04.049 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-03-27T10:17:04.049 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-03-27T10:17:04.049 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-03-27T10:17:04.049 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: Updating smithi161:/etc/ceph/ceph.client.admin.keyring 2023-03-27T10:17:04.049 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: Updating smithi161:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.client.admin.keyring 2023-03-27T10:17:04.049 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:04.049 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:04.050 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:04.050 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:04.050 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: Added host smithi171 2023-03-27T10:17:04.050 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:04.050 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:04.050 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:04.050 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-03-27T10:17:04.050 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:04.051 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/2402905318' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-03-27T10:17:04.051 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:04.051 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:04.051 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:04.051 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:04.051 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi171", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:17:04.051 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:04.051 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:04.052 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: Updating smithi171:/etc/ceph/ceph.conf 2023-03-27T10:17:04.052 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:04.052 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: Updating smithi171:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.conf 2023-03-27T10:17:04.052 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2023-03-27T10:17:04.052 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/2402905318' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-03-27T10:17:04.052 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: osdmap e4: 0 total, 0 up, 0 in 2023-03-27T10:17:04.052 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: Updating smithi171:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.client.admin.keyring 2023-03-27T10:17:04.052 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:04.053 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:04.053 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:04.053 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:04.053 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:04.053 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:04.053 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:04.053 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:04.054 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:04.054 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-03-27T10:17:04.054 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:04.054 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi159:172.21.15.159=a;smithi161:172.21.15.161=b;smithi171:172.21.15.171=c", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:17:04.054 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: Saving service mon spec with placement smithi159:172.21.15.159=a;smithi161:172.21.15.161=b;smithi171:172.21.15.171=c;count:3 2023-03-27T10:17:04.054 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: Deploying daemon mon.c on smithi171 2023-03-27T10:17:04.054 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:04.054 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: from='client.? 172.21.15.171:0/222368296' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-03-27T10:17:04.055 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-03-27T10:17:04.055 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: expand_channel_meta expand map: {default=false} 2023-03-27T10:17:04.055 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: expand_channel_meta from 'false' to 'false' 2023-03-27T10:17:04.055 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: expand_channel_meta expanded map: {default=false} 2023-03-27T10:17:04.055 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: expand_channel_meta expand map: {default=info} 2023-03-27T10:17:04.055 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: expand_channel_meta from 'info' to 'info' 2023-03-27T10:17:04.055 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: expand_channel_meta expanded map: {default=info} 2023-03-27T10:17:04.055 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: expand_channel_meta expand map: {default=daemon} 2023-03-27T10:17:04.056 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: expand_channel_meta from 'daemon' to 'daemon' 2023-03-27T10:17:04.056 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: expand_channel_meta expanded map: {default=daemon} 2023-03-27T10:17:04.056 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: expand_channel_meta expand map: {default=debug} 2023-03-27T10:17:04.056 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: expand_channel_meta from 'debug' to 'debug' 2023-03-27T10:17:04.056 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: expand_channel_meta expanded map: {default=debug} 2023-03-27T10:17:04.056 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:03 smithi171 ceph-mon[94808]: mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-03-27T10:17:04.378 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-03-27T10:17:04.379 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph mon dump -f json 2023-03-27T10:17:05.021 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.c/config 2023-03-27T10:17:09.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:17:09.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:09.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: mon.a calling monitor election 2023-03-27T10:17:09.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:09.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:09.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:09.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: mon.c calling monitor election 2023-03-27T10:17:09.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:09.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:09.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:09.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:09.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2023-03-27T10:17:09.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: monmap e2: 2 mons at {a=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0],c=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0]} removed_ranks: {} 2023-03-27T10:17:09.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: fsmap 2023-03-27T10:17:09.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: osdmap e4: 0 total, 0 up, 0 in 2023-03-27T10:17:09.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: mgrmap e13: x(active, since 41s) 2023-03-27T10:17:09.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: overall HEALTH_OK 2023-03-27T10:17:09.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:09.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:09.118 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-03-27T10:17:09.118 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:09.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:17:09.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:09.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: mon.a calling monitor election 2023-03-27T10:17:09.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:09.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:09.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:09.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: mon.c calling monitor election 2023-03-27T10:17:09.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:09.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:09.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:09.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:09.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2023-03-27T10:17:09.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: monmap e2: 2 mons at {a=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0],c=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0]} removed_ranks: {} 2023-03-27T10:17:09.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: fsmap 2023-03-27T10:17:09.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: osdmap e4: 0 total, 0 up, 0 in 2023-03-27T10:17:09.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: mgrmap e13: x(active, since 41s) 2023-03-27T10:17:09.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: overall HEALTH_OK 2023-03-27T10:17:09.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:09.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:09.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-03-27T10:17:09.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:09.243 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.244+0000 7fbd14c97700 1 -- 172.21.15.171:0/3666396981 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fbcf80035c0 msgr2=0x7fbcf8005a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:09.244 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.244+0000 7fbd14c97700 1 --2- 172.21.15.171:0/3666396981 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fbcf80035c0 0x7fbcf8005a40 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7fbd10100dc0 tx=0x7fbd0400fe60 comp rx=0 tx=0).stop 2023-03-27T10:17:09.244 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.244+0000 7fbd14c97700 1 -- 172.21.15.171:0/3666396981 shutdown_connections 2023-03-27T10:17:09.244 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.244+0000 7fbd14c97700 1 --2- 172.21.15.171:0/3666396981 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fbcf80035c0 0x7fbcf8005a40 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:09.244 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.244+0000 7fbd14c97700 1 -- 172.21.15.171:0/3666396981 >> 172.21.15.171:0/3666396981 conn(0x7fbd100fb450 msgr2=0x7fbd100fd860 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:17:09.244 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.244+0000 7fbd14c97700 1 -- 172.21.15.171:0/3666396981 shutdown_connections 2023-03-27T10:17:09.244 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.244+0000 7fbd14c97700 1 -- 172.21.15.171:0/3666396981 wait complete. 2023-03-27T10:17:09.245 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.245+0000 7fbd14c97700 1 Processor -- start 2023-03-27T10:17:09.246 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.246+0000 7fbd14c97700 1 -- start start 2023-03-27T10:17:09.246 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.246+0000 7fbd14c97700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fbcf80035c0 0x7fbd10194d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:09.246 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.246+0000 7fbd14c97700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fbd10195270 0x7fbd101996e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:09.246 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.246+0000 7fbd14c97700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd101957f0 con 0x7fbd10195270 2023-03-27T10:17:09.246 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.246+0000 7fbd14c97700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd10195960 con 0x7fbcf80035c0 2023-03-27T10:17:09.246 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.246+0000 7fbd0e59c700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fbcf80035c0 0x7fbd10194d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:09.246 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.246+0000 7fbd0e59c700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fbcf80035c0 0x7fbd10194d30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.171:3300/0 says I am v2:172.21.15.171:55058/0 (socket says 172.21.15.171:55058) 2023-03-27T10:17:09.247 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.246+0000 7fbd0e59c700 1 -- 172.21.15.171:0/3842540348 learned_addr learned my addr 172.21.15.171:0/3842540348 (peer_addr_for_me v2:172.21.15.171:0/0) 2023-03-27T10:17:09.247 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.246+0000 7fbd0e59c700 1 -- 172.21.15.171:0/3842540348 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fbcf80035c0 msgr2=0x7fbd10194d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 12 2023-03-27T10:17:09.247 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.246+0000 7fbd0dd9b700 1 --2- 172.21.15.171:0/3842540348 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fbd10195270 0x7fbd101996e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:09.247 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.246+0000 7fbd0e59c700 1 -- 172.21.15.171:0/3842540348 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fbcf80035c0 msgr2=0x7fbd10194d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2023-03-27T10:17:09.247 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.246+0000 7fbd0e59c700 1 --2- 172.21.15.171:0/3842540348 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fbcf80035c0 0x7fbd10194d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2023-03-27T10:17:09.247 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.246+0000 7fbd0e59c700 1 --2- 172.21.15.171:0/3842540348 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fbcf80035c0 0x7fbd10194d30 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-03-27T10:17:09.247 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.247+0000 7fbd0dd9b700 1 -- 172.21.15.171:0/3842540348 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fbcf80035c0 msgr2=0x7fbd10194d30 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-03-27T10:17:09.248 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.247+0000 7fbd0dd9b700 1 --2- 172.21.15.171:0/3842540348 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fbcf80035c0 0x7fbd10194d30 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:09.248 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.247+0000 7fbd0dd9b700 1 -- 172.21.15.171:0/3842540348 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbd0400f8b0 con 0x7fbd10195270 2023-03-27T10:17:09.248 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.247+0000 7fbd0dd9b700 1 --2- 172.21.15.171:0/3842540348 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fbd10195270 0x7fbd101996e0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7fbd0000b4d0 tx=0x7fbd0000b890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:17:09.249 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.247+0000 7fbcff7fe700 1 -- 172.21.15.171:0/3842540348 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 309+0+0 (secure 0 0 0) 0x7fbd00011630 con 0x7fbd10195270 2023-03-27T10:17:09.249 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.247+0000 7fbcff7fe700 1 -- 172.21.15.171:0/3842540348 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbd00011c50 con 0x7fbd10195270 2023-03-27T10:17:09.249 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.247+0000 7fbd14c97700 1 -- 172.21.15.171:0/3842540348 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbd101077a0 con 0x7fbd10195270 2023-03-27T10:17:09.249 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.247+0000 7fbcff7fe700 1 -- 172.21.15.171:0/3842540348 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 309+0+0 (secure 0 0 0) 0x7fbd00010d90 con 0x7fbd10195270 2023-03-27T10:17:09.250 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.247+0000 7fbd14c97700 1 -- 172.21.15.171:0/3842540348 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbd10107610 con 0x7fbd10195270 2023-03-27T10:17:09.250 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.248+0000 7fbcff7fe700 1 -- 172.21.15.171:0/3842540348 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 13) v1 ==== 43566+0+0 (secure 0 0 0) 0x7fbd00021070 con 0x7fbd10195270 2023-03-27T10:17:09.250 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.248+0000 7fbcff7fe700 1 --2- 172.21.15.171:0/3842540348 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fbcf0036710 0x7fbcf0038bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:09.250 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.248+0000 7fbcff7fe700 1 -- 172.21.15.171:0/3842540348 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fbd00048210 con 0x7fbd10195270 2023-03-27T10:17:09.250 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.249+0000 7fbd14c97700 1 -- 172.21.15.171:0/3842540348 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbd10107930 con 0x7fbd10195270 2023-03-27T10:17:09.250 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.249+0000 7fbd0e59c700 1 --2- 172.21.15.171:0/3842540348 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fbcf0036710 0x7fbcf0038bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:09.250 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.249+0000 7fbd0e59c700 1 --2- 172.21.15.171:0/3842540348 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fbcf0036710 0x7fbcf0038bc0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fbd040046a0 tx=0x7fbd04001e70 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:17:09.252 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.254+0000 7fbcff7fe700 1 -- 172.21.15.171:0/3842540348 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fbd0001a400 con 0x7fbd10195270 2023-03-27T10:17:09.510 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.511+0000 7fbd14c97700 1 -- 172.21.15.171:0/3842540348 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fbd101961d0 con 0x7fbd10195270 2023-03-27T10:17:09.510 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.512+0000 7fbcff7fe700 1 -- 172.21.15.171:0/3842540348 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 2 v2) v1 ==== 95+0+1010 (secure 0 0 0) 0x7fbd000172c0 con 0x7fbd10195270 2023-03-27T10:17:09.511 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:17:09.511 INFO:teuthology.orchestra.run.smithi171.stdout:{"epoch":2,"fsid":"2c3cd140-cc88-11ed-9afe-001a4aab830c","modified":"2023-03-27T10:17:03.751079Z","created":"2023-03-27T10:15:30.484420Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:3300","nonce":0},{"type":"v1","addr":"172.21.15.171:6789","nonce":0}]},"addr":"172.21.15.171:6789/0","public_addr":"172.21.15.171:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-03-27T10:17:09.512 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.513+0000 7fbd14c97700 1 -- 172.21.15.171:0/3842540348 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fbcf0036710 msgr2=0x7fbcf0038bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:09.512 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.513+0000 7fbd14c97700 1 --2- 172.21.15.171:0/3842540348 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fbcf0036710 0x7fbcf0038bc0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fbd040046a0 tx=0x7fbd04001e70 comp rx=0 tx=0).stop 2023-03-27T10:17:09.513 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.513+0000 7fbd14c97700 1 -- 172.21.15.171:0/3842540348 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fbd10195270 msgr2=0x7fbd101996e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:09.513 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.513+0000 7fbd14c97700 1 --2- 172.21.15.171:0/3842540348 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fbd10195270 0x7fbd101996e0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7fbd0000b4d0 tx=0x7fbd0000b890 comp rx=0 tx=0).stop 2023-03-27T10:17:09.513 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.513+0000 7fbd14c97700 1 -- 172.21.15.171:0/3842540348 shutdown_connections 2023-03-27T10:17:09.513 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.513+0000 7fbd14c97700 1 --2- 172.21.15.171:0/3842540348 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fbcf0036710 0x7fbcf0038bc0 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:09.513 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.513+0000 7fbd14c97700 1 --2- 172.21.15.171:0/3842540348 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fbcf80035c0 0x7fbd10194d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:09.513 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.513+0000 7fbd14c97700 1 --2- 172.21.15.171:0/3842540348 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fbd10195270 0x7fbd101996e0 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:09.514 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.513+0000 7fbd14c97700 1 -- 172.21.15.171:0/3842540348 >> 172.21.15.171:0/3842540348 conn(0x7fbd100fb450 msgr2=0x7fbd10103cf0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:17:09.514 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.513+0000 7fbd14c97700 1 -- 172.21.15.171:0/3842540348 shutdown_connections 2023-03-27T10:17:09.514 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:09.514+0000 7fbd14c97700 1 -- 172.21.15.171:0/3842540348 wait complete. 2023-03-27T10:17:09.514 INFO:teuthology.orchestra.run.smithi171.stderr:dumped monmap epoch 2 2023-03-27T10:17:10.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:09 smithi159 ceph-mon[93039]: Deploying daemon mon.b on smithi161 2023-03-27T10:17:10.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:09 smithi159 ceph-mon[93039]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:10.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:09 smithi159 ceph-mon[93039]: from='client.? 172.21.15.171:0/3842540348' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-03-27T10:17:10.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:09 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:10.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:09 smithi171 ceph-mon[94808]: Deploying daemon mon.b on smithi161 2023-03-27T10:17:10.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:09 smithi171 ceph-mon[94808]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:10.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:09 smithi171 ceph-mon[94808]: from='client.? 172.21.15.171:0/3842540348' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-03-27T10:17:10.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:09 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:10.983 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-03-27T10:17:10.984 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph mon dump -f json 2023-03-27T10:17:11.630 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.c/config 2023-03-27T10:17:11.674 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 systemd[1]: Starting Ceph mon.b for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:17:11.942 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 podman[94293]: 2023-03-27T10:17:11.942 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 bash[94293]: a292dc42b2f02a80a8d243fd463406b1fc74e265589bde69611f19cd35587f42 2023-03-27T10:17:11.942 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: set uid:gid to 167:167 (ceph:ceph) 2023-03-27T10:17:11.942 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: ceph version 18.0.0-2533-gb70fdf97 (b70fdf973c2ffa23c0d8aa6918519da05d922302) reef (dev), process ceph-mon, pid 7 2023-03-27T10:17:11.943 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: pidfile_write: ignore empty --pid-file 2023-03-27T10:17:11.943 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: load: jerasure load: lrc 2023-03-27T10:17:11.943 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: RocksDB version: 6.15.5 2023-03-27T10:17:11.943 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-03-27T10:17:11.943 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Compile date Mar 16 2023 2023-03-27T10:17:11.943 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: DB SUMMARY 2023-03-27T10:17:11.943 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: DB Session ID: 053R901N3TOAU9TJUZSC 2023-03-27T10:17:11.943 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: CURRENT file: CURRENT 2023-03-27T10:17:11.944 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: IDENTITY file: IDENTITY 2023-03-27T10:17:11.944 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-03-27T10:17:11.944 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-03-27T10:17:11.944 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2023-03-27T10:17:11.944 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.error_if_exists: 0 2023-03-27T10:17:11.944 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.create_if_missing: 0 2023-03-27T10:17:11.944 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.paranoid_checks: 1 2023-03-27T10:17:11.945 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-03-27T10:17:11.945 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.env: 0x56486b78d540 2023-03-27T10:17:11.945 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.fs: Posix File System 2023-03-27T10:17:11.945 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.info_log: 0x56486d2343a0 2023-03-27T10:17:11.945 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_file_opening_threads: 16 2023-03-27T10:17:11.946 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.statistics: (nil) 2023-03-27T10:17:11.946 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.use_fsync: 0 2023-03-27T10:17:11.946 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_log_file_size: 0 2023-03-27T10:17:11.946 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-03-27T10:17:11.946 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.log_file_time_to_roll: 0 2023-03-27T10:17:11.946 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.keep_log_file_num: 1000 2023-03-27T10:17:11.946 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.recycle_log_file_num: 0 2023-03-27T10:17:11.947 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.allow_fallocate: 1 2023-03-27T10:17:11.947 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.allow_mmap_reads: 0 2023-03-27T10:17:11.947 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.allow_mmap_writes: 0 2023-03-27T10:17:11.947 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.use_direct_reads: 0 2023-03-27T10:17:11.947 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-03-27T10:17:11.947 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.create_missing_column_families: 0 2023-03-27T10:17:11.947 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.db_log_dir: 2023-03-27T10:17:11.948 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-03-27T10:17:11.948 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.table_cache_numshardbits: 6 2023-03-27T10:17:11.948 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.WAL_ttl_seconds: 0 2023-03-27T10:17:11.948 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.WAL_size_limit_MB: 0 2023-03-27T10:17:11.948 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-03-27T10:17:11.948 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-03-27T10:17:11.948 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.is_fd_close_on_exec: 1 2023-03-27T10:17:11.948 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.advise_random_on_open: 1 2023-03-27T10:17:11.949 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.db_write_buffer_size: 0 2023-03-27T10:17:11.949 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.write_buffer_manager: 0x56486d4bcf60 2023-03-27T10:17:11.949 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-03-27T10:17:11.949 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-03-27T10:17:11.949 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-03-27T10:17:11.949 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.use_adaptive_mutex: 0 2023-03-27T10:17:11.949 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.rate_limiter: (nil) 2023-03-27T10:17:11.950 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-03-27T10:17:11.950 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.wal_recovery_mode: 2 2023-03-27T10:17:11.950 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.enable_thread_tracking: 0 2023-03-27T10:17:11.950 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.enable_pipelined_write: 0 2023-03-27T10:17:11.950 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.unordered_write: 0 2023-03-27T10:17:11.950 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-03-27T10:17:11.950 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-03-27T10:17:11.950 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-03-27T10:17:11.951 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-03-27T10:17:11.951 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.row_cache: None 2023-03-27T10:17:11.951 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.wal_filter: None 2023-03-27T10:17:11.951 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-03-27T10:17:11.951 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.allow_ingest_behind: 0 2023-03-27T10:17:11.951 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.preserve_deletes: 0 2023-03-27T10:17:11.951 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.two_write_queues: 0 2023-03-27T10:17:11.951 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.manual_wal_flush: 0 2023-03-27T10:17:11.952 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.atomic_flush: 0 2023-03-27T10:17:11.952 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-03-27T10:17:11.952 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.persist_stats_to_disk: 0 2023-03-27T10:17:11.952 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.write_dbid_to_manifest: 0 2023-03-27T10:17:11.952 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.log_readahead_size: 0 2023-03-27T10:17:11.952 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-03-27T10:17:11.952 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.best_efforts_recovery: 0 2023-03-27T10:17:11.952 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-03-27T10:17:11.953 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-03-27T10:17:11.953 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.allow_data_in_errors: 0 2023-03-27T10:17:11.953 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.db_host_id: __hostname__ 2023-03-27T10:17:11.954 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_background_jobs: 2 2023-03-27T10:17:11.954 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_background_compactions: -1 2023-03-27T10:17:11.954 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_subcompactions: 1 2023-03-27T10:17:11.955 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-03-27T10:17:11.955 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-03-27T10:17:11.955 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.delayed_write_rate : 16777216 2023-03-27T10:17:11.955 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_total_wal_size: 0 2023-03-27T10:17:11.955 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-03-27T10:17:11.955 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.stats_dump_period_sec: 600 2023-03-27T10:17:11.955 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.stats_persist_period_sec: 600 2023-03-27T10:17:11.956 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-03-27T10:17:11.956 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_open_files: -1 2023-03-27T10:17:11.956 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.bytes_per_sync: 0 2023-03-27T10:17:11.956 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.wal_bytes_per_sync: 0 2023-03-27T10:17:11.956 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.strict_bytes_per_sync: 0 2023-03-27T10:17:11.956 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compaction_readahead_size: 0 2023-03-27T10:17:11.956 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_background_flushes: -1 2023-03-27T10:17:11.956 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Compression algorithms supported: 2023-03-27T10:17:11.957 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-03-27T10:17:11.957 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: kZSTD supported: 0 2023-03-27T10:17:11.957 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: kXpressCompression supported: 0 2023-03-27T10:17:11.957 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: kLZ4HCCompression supported: 1 2023-03-27T10:17:11.957 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: kZlibCompression supported: 1 2023-03-27T10:17:11.957 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: kSnappyCompression supported: 1 2023-03-27T10:17:11.957 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: kLZ4Compression supported: 1 2023-03-27T10:17:11.958 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: kBZip2Compression supported: 0 2023-03-27T10:17:11.958 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Fast CRC32 supported: Supported on x86 2023-03-27T10:17:11.958 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2023-03-27T10:17:11.958 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-03-27T10:17:11.958 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-03-27T10:17:11.958 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.merge_operator: 2023-03-27T10:17:11.958 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compaction_filter: None 2023-03-27T10:17:11.958 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compaction_filter_factory: None 2023-03-27T10:17:11.959 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.sst_partitioner_factory: None 2023-03-27T10:17:11.959 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.memtable_factory: SkipListFactory 2023-03-27T10:17:11.959 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.table_factory: BlockBasedTable 2023-03-27T10:17:11.959 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56486d2445f0) 2023-03-27T10:17:11.959 INFO:journalctl@ceph.mon.b.smithi161.stdout: cache_index_and_filter_blocks: 1 2023-03-27T10:17:11.959 INFO:journalctl@ceph.mon.b.smithi161.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-03-27T10:17:11.959 INFO:journalctl@ceph.mon.b.smithi161.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-03-27T10:17:11.959 INFO:journalctl@ceph.mon.b.smithi161.stdout: pin_top_level_index_and_filter: 1 2023-03-27T10:17:11.960 INFO:journalctl@ceph.mon.b.smithi161.stdout: index_type: 0 2023-03-27T10:17:11.960 INFO:journalctl@ceph.mon.b.smithi161.stdout: data_block_index_type: 0 2023-03-27T10:17:11.960 INFO:journalctl@ceph.mon.b.smithi161.stdout: index_shortening: 1 2023-03-27T10:17:11.960 INFO:journalctl@ceph.mon.b.smithi161.stdout: data_block_hash_table_util_ratio: 0.750000 2023-03-27T10:17:11.960 INFO:journalctl@ceph.mon.b.smithi161.stdout: hash_index_allow_collision: 1 2023-03-27T10:17:11.960 INFO:journalctl@ceph.mon.b.smithi161.stdout: checksum: 1 2023-03-27T10:17:11.960 INFO:journalctl@ceph.mon.b.smithi161.stdout: no_block_cache: 0 2023-03-27T10:17:11.960 INFO:journalctl@ceph.mon.b.smithi161.stdout: block_cache: 0x56486d285350 2023-03-27T10:17:11.961 INFO:journalctl@ceph.mon.b.smithi161.stdout: block_cache_name: BinnedLRUCache 2023-03-27T10:17:11.961 INFO:journalctl@ceph.mon.b.smithi161.stdout: block_cache_options: 2023-03-27T10:17:11.961 INFO:journalctl@ceph.mon.b.smithi161.stdout: capacity : 536870912 2023-03-27T10:17:11.961 INFO:journalctl@ceph.mon.b.smithi161.stdout: num_shard_bits : 4 2023-03-27T10:17:11.961 INFO:journalctl@ceph.mon.b.smithi161.stdout: strict_capacity_limit : 0 2023-03-27T10:17:11.961 INFO:journalctl@ceph.mon.b.smithi161.stdout: high_pri_pool_ratio: 0.000 2023-03-27T10:17:11.961 INFO:journalctl@ceph.mon.b.smithi161.stdout: block_cache_compressed: (nil) 2023-03-27T10:17:11.962 INFO:journalctl@ceph.mon.b.smithi161.stdout: persistent_cache: (nil) 2023-03-27T10:17:11.962 INFO:journalctl@ceph.mon.b.smithi161.stdout: block_size: 4096 2023-03-27T10:17:11.962 INFO:journalctl@ceph.mon.b.smithi161.stdout: block_size_deviation: 10 2023-03-27T10:17:11.962 INFO:journalctl@ceph.mon.b.smithi161.stdout: block_restart_interval: 16 2023-03-27T10:17:11.962 INFO:journalctl@ceph.mon.b.smithi161.stdout: index_block_restart_interval: 1 2023-03-27T10:17:11.962 INFO:journalctl@ceph.mon.b.smithi161.stdout: metadata_block_size: 4096 2023-03-27T10:17:11.962 INFO:journalctl@ceph.mon.b.smithi161.stdout: partition_filters: 0 2023-03-27T10:17:11.962 INFO:journalctl@ceph.mon.b.smithi161.stdout: use_delta_encoding: 1 2023-03-27T10:17:11.963 INFO:journalctl@ceph.mon.b.smithi161.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-03-27T10:17:11.963 INFO:journalctl@ceph.mon.b.smithi161.stdout: whole_key_filtering: 1 2023-03-27T10:17:11.963 INFO:journalctl@ceph.mon.b.smithi161.stdout: verify_compression: 0 2023-03-27T10:17:11.963 INFO:journalctl@ceph.mon.b.smithi161.stdout: read_amp_bytes_per_bit: 0 2023-03-27T10:17:11.963 INFO:journalctl@ceph.mon.b.smithi161.stdout: format_version: 4 2023-03-27T10:17:11.963 INFO:journalctl@ceph.mon.b.smithi161.stdout: enable_index_compression: 1 2023-03-27T10:17:11.963 INFO:journalctl@ceph.mon.b.smithi161.stdout: block_align: 0 2023-03-27T10:17:11.963 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.write_buffer_size: 33554432 2023-03-27T10:17:11.964 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_write_buffer_number: 2 2023-03-27T10:17:11.964 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compression: NoCompression 2023-03-27T10:17:11.964 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.bottommost_compression: Disabled 2023-03-27T10:17:11.964 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.prefix_extractor: nullptr 2023-03-27T10:17:11.964 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-03-27T10:17:11.964 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.num_levels: 7 2023-03-27T10:17:11.964 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-03-27T10:17:11.964 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-03-27T10:17:11.965 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-03-27T10:17:11.965 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-03-27T10:17:11.965 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-03-27T10:17:11.965 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-03-27T10:17:11.965 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-03-27T10:17:11.965 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-03-27T10:17:11.965 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-03-27T10:17:11.966 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-03-27T10:17:11.966 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compression_opts.window_bits: -14 2023-03-27T10:17:11.966 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compression_opts.level: 32767 2023-03-27T10:17:11.966 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compression_opts.strategy: 0 2023-03-27T10:17:11.966 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-03-27T10:17:11.966 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-03-27T10:17:11.966 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-03-27T10:17:11.966 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compression_opts.enabled: false 2023-03-27T10:17:11.967 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-03-27T10:17:11.967 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-03-27T10:17:11.967 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-03-27T10:17:11.967 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.target_file_size_base: 67108864 2023-03-27T10:17:11.967 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.target_file_size_multiplier: 1 2023-03-27T10:17:11.967 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-03-27T10:17:11.967 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-03-27T10:17:11.967 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-03-27T10:17:11.968 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-03-27T10:17:11.968 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-03-27T10:17:11.968 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-03-27T10:17:11.968 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-03-27T10:17:11.968 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-03-27T10:17:11.968 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-03-27T10:17:11.968 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-03-27T10:17:11.968 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-03-27T10:17:11.969 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-03-27T10:17:11.969 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.arena_block_size: 4194304 2023-03-27T10:17:11.969 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-03-27T10:17:11.969 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-03-27T10:17:11.969 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-03-27T10:17:11.969 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.disable_auto_compactions: 0 2023-03-27T10:17:11.969 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-03-27T10:17:11.970 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-03-27T10:17:11.970 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-03-27T10:17:11.970 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-03-27T10:17:11.970 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-03-27T10:17:11.970 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-03-27T10:17:11.970 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-03-27T10:17:11.970 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-03-27T10:17:11.970 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-03-27T10:17:11.971 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-03-27T10:17:11.971 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.table_properties_collectors: 2023-03-27T10:17:11.971 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.inplace_update_support: 0 2023-03-27T10:17:11.971 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.inplace_update_num_locks: 10000 2023-03-27T10:17:11.971 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-03-27T10:17:11.971 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-03-27T10:17:11.971 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.memtable_huge_page_size: 0 2023-03-27T10:17:11.971 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.bloom_locality: 0 2023-03-27T10:17:11.972 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.max_successive_merges: 0 2023-03-27T10:17:11.972 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.optimize_filters_for_hits: 0 2023-03-27T10:17:11.972 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.paranoid_file_checks: 0 2023-03-27T10:17:11.972 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.force_consistency_checks: 1 2023-03-27T10:17:11.972 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.report_bg_io_stats: 0 2023-03-27T10:17:11.972 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.ttl: 2592000 2023-03-27T10:17:11.972 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.periodic_compaction_seconds: 0 2023-03-27T10:17:11.972 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.enable_blob_files: false 2023-03-27T10:17:11.973 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.min_blob_size: 0 2023-03-27T10:17:11.973 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.blob_file_size: 268435456 2023-03-27T10:17:11.973 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.blob_compression_type: NoCompression 2023-03-27T10:17:11.973 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.enable_blob_garbage_collection: false 2023-03-27T10:17:11.973 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-03-27T10:17:11.973 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-03-27T10:17:11.973 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-03-27T10:17:11.973 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-03-27T10:17:11.974 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: EVENT_LOG_v1 {"time_micros": 1679912231867304, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-03-27T10:17:11.974 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-03-27T10:17:11.974 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-03-27T10:17:11.974 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: EVENT_LOG_v1 {"time_micros": 1679912231867573, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1679912231, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "b83d92c2-b0ac-4949-ac06-6bb2a04dd813", "db_session_id": "053R901N3TOAU9TJUZSC"}} 2023-03-27T10:17:11.974 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-03-27T10:17:11.974 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: EVENT_LOG_v1 {"time_micros": 1679912231867888, "job": 1, "event": "recovery_finished"} 2023-03-27T10:17:11.974 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-03-27T10:17:11.975 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x56486d219c00 2023-03-27T10:17:11.975 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: DB pointer 0x56486d236000 2023-03-27T10:17:11.975 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-03-27T10:17:11.975 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-03-27T10:17:11.975 INFO:journalctl@ceph.mon.b.smithi161.stdout: ** DB Stats ** 2023-03-27T10:17:11.975 INFO:journalctl@ceph.mon.b.smithi161.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-03-27T10:17:11.975 INFO:journalctl@ceph.mon.b.smithi161.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-03-27T10:17:11.975 INFO:journalctl@ceph.mon.b.smithi161.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-03-27T10:17:11.976 INFO:journalctl@ceph.mon.b.smithi161.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-03-27T10:17:11.976 INFO:journalctl@ceph.mon.b.smithi161.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-03-27T10:17:11.976 INFO:journalctl@ceph.mon.b.smithi161.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-03-27T10:17:11.976 INFO:journalctl@ceph.mon.b.smithi161.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-03-27T10:17:11.976 INFO:journalctl@ceph.mon.b.smithi161.stdout: 2023-03-27T10:17:11.976 INFO:journalctl@ceph.mon.b.smithi161.stdout: ** Compaction Stats [default] ** 2023-03-27T10:17:11.976 INFO:journalctl@ceph.mon.b.smithi161.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-03-27T10:17:11.976 INFO:journalctl@ceph.mon.b.smithi161.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-03-27T10:17:11.977 INFO:journalctl@ceph.mon.b.smithi161.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-03-27T10:17:11.977 INFO:journalctl@ceph.mon.b.smithi161.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-03-27T10:17:11.977 INFO:journalctl@ceph.mon.b.smithi161.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-03-27T10:17:11.977 INFO:journalctl@ceph.mon.b.smithi161.stdout: 2023-03-27T10:17:11.977 INFO:journalctl@ceph.mon.b.smithi161.stdout: ** Compaction Stats [default] ** 2023-03-27T10:17:11.977 INFO:journalctl@ceph.mon.b.smithi161.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-03-27T10:17:11.977 INFO:journalctl@ceph.mon.b.smithi161.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-03-27T10:17:11.977 INFO:journalctl@ceph.mon.b.smithi161.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-03-27T10:17:11.978 INFO:journalctl@ceph.mon.b.smithi161.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-03-27T10:17:11.978 INFO:journalctl@ceph.mon.b.smithi161.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-03-27T10:17:11.978 INFO:journalctl@ceph.mon.b.smithi161.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-03-27T10:17:11.978 INFO:journalctl@ceph.mon.b.smithi161.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-03-27T10:17:11.978 INFO:journalctl@ceph.mon.b.smithi161.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-03-27T10:17:11.978 INFO:journalctl@ceph.mon.b.smithi161.stdout: AddFile(Keys): cumulative 0, interval 0 2023-03-27T10:17:11.978 INFO:journalctl@ceph.mon.b.smithi161.stdout: Cumulative compaction: 0.00 GB write, 0.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-03-27T10:17:11.979 INFO:journalctl@ceph.mon.b.smithi161.stdout: Interval compaction: 0.00 GB write, 0.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-03-27T10:17:11.979 INFO:journalctl@ceph.mon.b.smithi161.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-03-27T10:17:11.979 INFO:journalctl@ceph.mon.b.smithi161.stdout: 2023-03-27T10:17:11.979 INFO:journalctl@ceph.mon.b.smithi161.stdout: ** File Read Latency Histogram By Level [default] ** 2023-03-27T10:17:11.979 INFO:journalctl@ceph.mon.b.smithi161.stdout: 2023-03-27T10:17:11.979 INFO:journalctl@ceph.mon.b.smithi161.stdout: ** Compaction Stats [default] ** 2023-03-27T10:17:11.979 INFO:journalctl@ceph.mon.b.smithi161.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-03-27T10:17:11.979 INFO:journalctl@ceph.mon.b.smithi161.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-03-27T10:17:11.980 INFO:journalctl@ceph.mon.b.smithi161.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-03-27T10:17:11.980 INFO:journalctl@ceph.mon.b.smithi161.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-03-27T10:17:11.980 INFO:journalctl@ceph.mon.b.smithi161.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-03-27T10:17:11.980 INFO:journalctl@ceph.mon.b.smithi161.stdout: 2023-03-27T10:17:11.980 INFO:journalctl@ceph.mon.b.smithi161.stdout: ** Compaction Stats [default] ** 2023-03-27T10:17:11.980 INFO:journalctl@ceph.mon.b.smithi161.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-03-27T10:17:11.980 INFO:journalctl@ceph.mon.b.smithi161.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-03-27T10:17:11.980 INFO:journalctl@ceph.mon.b.smithi161.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-03-27T10:17:11.981 INFO:journalctl@ceph.mon.b.smithi161.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-03-27T10:17:11.981 INFO:journalctl@ceph.mon.b.smithi161.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-03-27T10:17:11.981 INFO:journalctl@ceph.mon.b.smithi161.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-03-27T10:17:11.981 INFO:journalctl@ceph.mon.b.smithi161.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-03-27T10:17:11.981 INFO:journalctl@ceph.mon.b.smithi161.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-03-27T10:17:11.981 INFO:journalctl@ceph.mon.b.smithi161.stdout: AddFile(Keys): cumulative 0, interval 0 2023-03-27T10:17:11.981 INFO:journalctl@ceph.mon.b.smithi161.stdout: Cumulative compaction: 0.00 GB write, 0.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-03-27T10:17:11.981 INFO:journalctl@ceph.mon.b.smithi161.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-03-27T10:17:11.982 INFO:journalctl@ceph.mon.b.smithi161.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-03-27T10:17:11.982 INFO:journalctl@ceph.mon.b.smithi161.stdout: 2023-03-27T10:17:11.982 INFO:journalctl@ceph.mon.b.smithi161.stdout: ** File Read Latency Histogram By Level [default] ** 2023-03-27T10:17:11.982 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.b does not exist in monmap, will attempt to join an existing cluster 2023-03-27T10:17:11.982 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: using public_addr v2:172.21.15.161:0/0 -> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] 2023-03-27T10:17:11.982 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: starting mon.b rank -1 at public addrs [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] at bind addrs [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:17:11.982 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.b@-1(???) e0 preinit fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:17:11.982 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.b@-1(synchronizing).mds e1 new map 2023-03-27T10:17:11.983 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.b@-1(synchronizing).mds e1 print_map 2023-03-27T10:17:11.983 INFO:journalctl@ceph.mon.b.smithi161.stdout: e1 2023-03-27T10:17:11.983 INFO:journalctl@ceph.mon.b.smithi161.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-03-27T10:17:11.983 INFO:journalctl@ceph.mon.b.smithi161.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-03-27T10:17:11.983 INFO:journalctl@ceph.mon.b.smithi161.stdout: legacy client fscid: -1 2023-03-27T10:17:11.983 INFO:journalctl@ceph.mon.b.smithi161.stdout: 2023-03-27T10:17:11.983 INFO:journalctl@ceph.mon.b.smithi161.stdout: No filesystems configured 2023-03-27T10:17:11.984 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-03-27T10:17:11.984 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-03-27T10:17:11.984 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-03-27T10:17:11.984 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-03-27T10:17:11.984 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-03-27T10:17:11.984 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-03-27T10:17:11.984 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-03-27T10:17:11.984 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-03-27T10:17:11.985 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-03-27T10:17:11.985 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-03-27T10:17:11.985 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: Updating smithi161:/etc/ceph/ceph.client.admin.keyring 2023-03-27T10:17:11.985 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: Updating smithi161:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.client.admin.keyring 2023-03-27T10:17:11.985 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:11.985 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:11.985 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:11.985 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:11.986 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: Added host smithi171 2023-03-27T10:17:11.986 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:11.986 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:11.986 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:11.986 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-03-27T10:17:11.986 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:11.986 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/2402905318' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-03-27T10:17:11.986 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:11.987 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:11.987 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:11.987 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:11.987 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi171", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:17:11.987 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:11.987 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:11.987 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: Updating smithi171:/etc/ceph/ceph.conf 2023-03-27T10:17:11.987 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:11.988 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: Updating smithi171:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.conf 2023-03-27T10:17:11.988 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2023-03-27T10:17:11.988 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/2402905318' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-03-27T10:17:11.988 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: osdmap e4: 0 total, 0 up, 0 in 2023-03-27T10:17:11.988 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: Updating smithi171:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.client.admin.keyring 2023-03-27T10:17:11.988 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:11.988 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:11.988 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:11.989 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:11.989 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:11.989 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:11.989 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:11.989 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:11.989 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:11.989 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-03-27T10:17:11.990 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:11.990 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi159:172.21.15.159=a;smithi161:172.21.15.161=b;smithi171:172.21.15.171=c", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:17:11.990 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: Saving service mon spec with placement smithi159:172.21.15.159=a;smithi161:172.21.15.161=b;smithi171:172.21.15.171=c;count:3 2023-03-27T10:17:11.990 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: Deploying daemon mon.c on smithi171 2023-03-27T10:17:11.990 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:11.990 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='client.? 172.21.15.171:0/222368296' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-03-27T10:17:11.990 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:17:11.990 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:11.991 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.a calling monitor election 2023-03-27T10:17:11.991 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:11.991 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:11.991 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:11.991 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.c calling monitor election 2023-03-27T10:17:11.991 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:11.991 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:11.991 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:11.992 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:11.992 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2023-03-27T10:17:11.992 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: monmap e2: 2 mons at {a=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0],c=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0]} removed_ranks: {} 2023-03-27T10:17:11.992 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: fsmap 2023-03-27T10:17:11.992 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: osdmap e4: 0 total, 0 up, 0 in 2023-03-27T10:17:11.992 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mgrmap e13: x(active, since 41s) 2023-03-27T10:17:11.992 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: overall HEALTH_OK 2023-03-27T10:17:11.992 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:11.993 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:11.993 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-03-27T10:17:11.993 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:11.993 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: Deploying daemon mon.b on smithi161 2023-03-27T10:17:11.993 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:11.993 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='client.? 172.21.15.171:0/3842540348' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-03-27T10:17:11.993 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:11.994 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-03-27T10:17:11.994 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: expand_channel_meta expand map: {default=false} 2023-03-27T10:17:11.994 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: expand_channel_meta from 'false' to 'false' 2023-03-27T10:17:11.994 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: expand_channel_meta expanded map: {default=false} 2023-03-27T10:17:11.994 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: expand_channel_meta expand map: {default=info} 2023-03-27T10:17:11.994 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: expand_channel_meta from 'info' to 'info' 2023-03-27T10:17:11.994 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: expand_channel_meta expanded map: {default=info} 2023-03-27T10:17:11.994 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: expand_channel_meta expand map: {default=daemon} 2023-03-27T10:17:11.995 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: expand_channel_meta from 'daemon' to 'daemon' 2023-03-27T10:17:11.995 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: expand_channel_meta expanded map: {default=daemon} 2023-03-27T10:17:11.995 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: expand_channel_meta expand map: {default=debug} 2023-03-27T10:17:11.995 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: expand_channel_meta from 'debug' to 'debug' 2023-03-27T10:17:11.995 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: expand_channel_meta expanded map: {default=debug} 2023-03-27T10:17:11.995 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 ceph-mon[94333]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-03-27T10:17:11.995 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:11 smithi161 systemd[1]: Started Ceph mon.b for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:17:17.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: mon.c calling monitor election 2023-03-27T10:17:17.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:17:17.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: mon.a calling monitor election 2023-03-27T10:17:17.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:17.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:17.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:17.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:17.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:17.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:17.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:17.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:17.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:17.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2023-03-27T10:17:17.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: monmap e3: 3 mons at {a=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0],b=[v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0],c=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0]} removed_ranks: {} 2023-03-27T10:17:17.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: fsmap 2023-03-27T10:17:17.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: osdmap e4: 0 total, 0 up, 0 in 2023-03-27T10:17:17.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: mgrmap e13: x(active, since 50s) 2023-03-27T10:17:17.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2023-03-27T10:17:17.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2023-03-27T10:17:17.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2023-03-27T10:17:17.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: mon.b (rank 2) addr [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] is down (out of quorum) 2023-03-27T10:17:17.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:17.230 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:16 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:17.281 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.281+0000 7fe324566700 1 -- 172.21.15.171:0/3828382267 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe31c104850 msgr2=0x7fe308007820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:17.281 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.281+0000 7fe324566700 1 --2- 172.21.15.171:0/3828382267 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe31c104850 0x7fe308007820 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7fe30c00fc30 tx=0x7fe30c00ff40 comp rx=0 tx=0).stop 2023-03-27T10:17:17.281 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.281+0000 7fe324566700 1 -- 172.21.15.171:0/3828382267 shutdown_connections 2023-03-27T10:17:17.281 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.281+0000 7fe324566700 1 --2- 172.21.15.171:0/3828382267 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe31c104850 0x7fe308007820 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:17.281 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.281+0000 7fe324566700 1 -- 172.21.15.171:0/3828382267 >> 172.21.15.171:0/3828382267 conn(0x7fe31c0fa370 msgr2=0x7fe31c0fc780 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:17:17.282 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.282+0000 7fe324566700 1 -- 172.21.15.171:0/3828382267 shutdown_connections 2023-03-27T10:17:17.282 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.282+0000 7fe324566700 1 -- 172.21.15.171:0/3828382267 wait complete. 2023-03-27T10:17:17.284 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.282+0000 7fe324566700 1 Processor -- start 2023-03-27T10:17:17.284 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.282+0000 7fe324566700 1 -- start start 2023-03-27T10:17:17.284 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.282+0000 7fe324566700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe31c104850 0x7fe31c192d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:17.284 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe324566700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe31c193280 0x7fe31c1976f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:17.284 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe324566700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe31c197c30 0x7fe31c1980d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:17.284 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe324566700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe31c193950 con 0x7fe31c197c30 2023-03-27T10:17:17.284 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe324566700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe31c19f9a0 con 0x7fe31c104850 2023-03-27T10:17:17.285 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe324566700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe31c19fae0 con 0x7fe31c193280 2023-03-27T10:17:17.285 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe321b01700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe31c193280 0x7fe31c1976f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:17.285 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe321b01700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe31c193280 0x7fe31c1976f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.171:3300/0 says I am v2:172.21.15.171:55088/0 (socket says 172.21.15.171:55088) 2023-03-27T10:17:17.285 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe321b01700 1 -- 172.21.15.171:0/1736572942 learned_addr learned my addr 172.21.15.171:0/1736572942 (peer_addr_for_me v2:172.21.15.171:0/0) 2023-03-27T10:17:17.285 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe322302700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe31c104850 0x7fe31c192d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:17.285 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe321b01700 1 -- 172.21.15.171:0/1736572942 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe31c193280 msgr2=0x7fe31c1976f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 13 2023-03-27T10:17:17.285 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe321b01700 1 -- 172.21.15.171:0/1736572942 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe31c193280 msgr2=0x7fe31c1976f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2023-03-27T10:17:17.286 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe321b01700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe31c193280 0x7fe31c1976f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2023-03-27T10:17:17.286 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe322b03700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe31c197c30 0x7fe31c1980d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:17.286 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe321b01700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe31c193280 0x7fe31c1976f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-03-27T10:17:17.286 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe322302700 1 -- 172.21.15.171:0/1736572942 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe31c104850 msgr2=0x7fe31c192d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 12 2023-03-27T10:17:17.286 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe322302700 1 -- 172.21.15.171:0/1736572942 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe31c104850 msgr2=0x7fe31c192d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2023-03-27T10:17:17.286 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe322302700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe31c104850 0x7fe31c192d40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2023-03-27T10:17:17.286 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe322302700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe31c104850 0x7fe31c192d40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-03-27T10:17:17.287 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe322b03700 1 -- 172.21.15.171:0/1736572942 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe31c193280 msgr2=0x7fe31c1976f0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-03-27T10:17:17.287 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe322b03700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe31c193280 0x7fe31c1976f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:17.287 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe322b03700 1 -- 172.21.15.171:0/1736572942 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe31c104850 msgr2=0x7fe31c192d40 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-03-27T10:17:17.287 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe322b03700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe31c104850 0x7fe31c192d40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:17.287 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe322b03700 1 -- 172.21.15.171:0/1736572942 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe30c00f910 con 0x7fe31c197c30 2023-03-27T10:17:17.287 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe322b03700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe31c197c30 0x7fe31c1980d0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7fe31400bd80 tx=0x7fe31400be60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:17:17.287 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.283+0000 7fe3137fe700 1 -- 172.21.15.171:0/1736572942 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe31400cd10 con 0x7fe31c197c30 2023-03-27T10:17:17.287 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.284+0000 7fe3137fe700 1 -- 172.21.15.171:0/1736572942 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe31400ce70 con 0x7fe31c197c30 2023-03-27T10:17:17.288 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.284+0000 7fe324566700 1 -- 172.21.15.171:0/1736572942 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe31c105660 con 0x7fe31c197c30 2023-03-27T10:17:17.288 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.284+0000 7fe3137fe700 1 -- 172.21.15.171:0/1736572942 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe314012880 con 0x7fe31c197c30 2023-03-27T10:17:17.288 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.284+0000 7fe324566700 1 -- 172.21.15.171:0/1736572942 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe31c069480 con 0x7fe31c197c30 2023-03-27T10:17:17.288 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.284+0000 7fe3137fe700 1 -- 172.21.15.171:0/1736572942 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 13) v1 ==== 43566+0+0 (secure 0 0 0) 0x7fe31401b030 con 0x7fe31c197c30 2023-03-27T10:17:17.288 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.284+0000 7fe3137fe700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe300036da0 0x7fe300039250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:17.288 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.284+0000 7fe3137fe700 1 -- 172.21.15.171:0/1736572942 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fe314049210 con 0x7fe31c197c30 2023-03-27T10:17:17.288 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.285+0000 7fe322302700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe300036da0 0x7fe300039250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:17.288 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.285+0000 7fe324566700 1 -- 172.21.15.171:0/1736572942 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe31c1057f0 con 0x7fe31c197c30 2023-03-27T10:17:17.289 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.285+0000 7fe322302700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe300036da0 0x7fe300039250 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fe30c000c00 tx=0x7fe30c014040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:17:17.289 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.290+0000 7fe3137fe700 1 -- 172.21.15.171:0/1736572942 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fe314019030 con 0x7fe31c197c30 2023-03-27T10:17:17.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: mon.c calling monitor election 2023-03-27T10:17:17.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:17:17.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: mon.a calling monitor election 2023-03-27T10:17:17.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:17.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:17.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:17.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:17.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:17.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:17.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:17.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:17.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:17.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2023-03-27T10:17:17.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: monmap e3: 3 mons at {a=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0],b=[v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0],c=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0]} removed_ranks: {} 2023-03-27T10:17:17.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: fsmap 2023-03-27T10:17:17.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: osdmap e4: 0 total, 0 up, 0 in 2023-03-27T10:17:17.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: mgrmap e13: x(active, since 50s) 2023-03-27T10:17:17.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2023-03-27T10:17:17.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2023-03-27T10:17:17.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2023-03-27T10:17:17.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: mon.b (rank 2) addr [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] is down (out of quorum) 2023-03-27T10:17:17.368 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:17.368 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:16 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:17.554 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.555+0000 7fe324566700 1 -- 172.21.15.171:0/1736572942 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fe31c1a05c0 con 0x7fe31c197c30 2023-03-27T10:17:17.555 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.556+0000 7fe3137fe700 1 -- 172.21.15.171:0/1736572942 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 3 v3) v1 ==== 95+0+1278 (secure 0 0 0) 0x7fe314011690 con 0x7fe31c197c30 2023-03-27T10:17:17.555 INFO:teuthology.orchestra.run.smithi171.stdout: 2023-03-27T10:17:17.555 INFO:teuthology.orchestra.run.smithi171.stdout:{"epoch":3,"fsid":"2c3cd140-cc88-11ed-9afe-001a4aab830c","modified":"2023-03-27T10:17:11.891558Z","created":"2023-03-27T10:15:30.484420Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:3300","nonce":0},{"type":"v1","addr":"172.21.15.171:6789","nonce":0}]},"addr":"172.21.15.171:6789/0","public_addr":"172.21.15.171:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:3300","nonce":0},{"type":"v1","addr":"172.21.15.161:6789","nonce":0}]},"addr":"172.21.15.161:6789/0","public_addr":"172.21.15.161:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-03-27T10:17:17.557 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.558+0000 7fe324566700 1 -- 172.21.15.171:0/1736572942 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe300036da0 msgr2=0x7fe300039250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:17.557 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.558+0000 7fe324566700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe300036da0 0x7fe300039250 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fe30c000c00 tx=0x7fe30c014040 comp rx=0 tx=0).stop 2023-03-27T10:17:17.557 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.558+0000 7fe324566700 1 -- 172.21.15.171:0/1736572942 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe31c197c30 msgr2=0x7fe31c1980d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:17.558 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.558+0000 7fe324566700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe31c197c30 0x7fe31c1980d0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7fe31400bd80 tx=0x7fe31400be60 comp rx=0 tx=0).stop 2023-03-27T10:17:17.558 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.558+0000 7fe324566700 1 -- 172.21.15.171:0/1736572942 shutdown_connections 2023-03-27T10:17:17.558 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.558+0000 7fe324566700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe300036da0 0x7fe300039250 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:17.558 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.558+0000 7fe324566700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe31c197c30 0x7fe31c1980d0 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:17.558 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.558+0000 7fe324566700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe31c104850 0x7fe31c192d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:17.558 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.558+0000 7fe324566700 1 --2- 172.21.15.171:0/1736572942 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe31c193280 0x7fe31c1976f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:17.558 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.558+0000 7fe324566700 1 -- 172.21.15.171:0/1736572942 >> 172.21.15.171:0/1736572942 conn(0x7fe31c0fa370 msgr2=0x7fe31c102c10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:17:17.558 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.558+0000 7fe324566700 1 -- 172.21.15.171:0/1736572942 shutdown_connections 2023-03-27T10:17:17.559 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:17.558+0000 7fe324566700 1 -- 172.21.15.171:0/1736572942 wait complete. 2023-03-27T10:17:17.559 INFO:teuthology.orchestra.run.smithi171.stderr:dumped monmap epoch 3 2023-03-27T10:17:17.996 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-03-27T10:17:17.996 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph config generate-minimal-conf 2023-03-27T10:17:18.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:18 smithi159 ceph-mon[93039]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:18.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:18.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:18.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:18 smithi159 ceph-mon[93039]: from='client.? 172.21.15.171:0/1736572942' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-03-27T10:17:18.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:18.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:18 smithi171 ceph-mon[94808]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:18.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:18.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:18.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:18 smithi171 ceph-mon[94808]: from='client.? 172.21.15.171:0/1736572942' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-03-27T10:17:18.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:18.646 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:17:19.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: mon.c calling monitor election 2023-03-27T10:17:19.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:17:19.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: mon.a calling monitor election 2023-03-27T10:17:19.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:19.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:17:19.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:19.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:19.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:19.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:19.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:19.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:19.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:19.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2023-03-27T10:17:19.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: monmap e3: 3 mons at {a=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0],b=[v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0],c=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0]} removed_ranks: {} 2023-03-27T10:17:19.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: fsmap 2023-03-27T10:17:19.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: osdmap e4: 0 total, 0 up, 0 in 2023-03-27T10:17:19.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: mgrmap e13: x(active, since 50s) 2023-03-27T10:17:19.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2023-03-27T10:17:19.337 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2023-03-27T10:17:19.337 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2023-03-27T10:17:19.337 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: mon.b (rank 2) addr [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] is down (out of quorum) 2023-03-27T10:17:19.337 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:19.337 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:19.337 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:19.337 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:19.337 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:19.338 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: from='client.? 172.21.15.171:0/1736572942' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-03-27T10:17:19.338 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:19.864 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.865+0000 7f5b40a7f700 1 -- 172.21.15.159:0/1735768281 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5b3c105810 msgr2=0x7f5b3c105be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:19.865 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.865+0000 7f5b40a7f700 1 --2- 172.21.15.159:0/1735768281 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5b3c105810 0x7f5b3c105be0 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f5b3400b0a0 tx=0x7f5b3400b3b0 comp rx=0 tx=0).stop 2023-03-27T10:17:19.865 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.865+0000 7f5b40a7f700 1 -- 172.21.15.159:0/1735768281 shutdown_connections 2023-03-27T10:17:19.865 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.865+0000 7f5b40a7f700 1 --2- 172.21.15.159:0/1735768281 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5b3c105810 0x7f5b3c105be0 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:19.865 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.865+0000 7f5b40a7f700 1 -- 172.21.15.159:0/1735768281 >> 172.21.15.159:0/1735768281 conn(0x7f5b3c0fb330 msgr2=0x7f5b3c0fd740 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:17:19.865 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.865+0000 7f5b40a7f700 1 -- 172.21.15.159:0/1735768281 shutdown_connections 2023-03-27T10:17:19.865 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.865+0000 7f5b40a7f700 1 -- 172.21.15.159:0/1735768281 wait complete. 2023-03-27T10:17:19.867 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.866+0000 7f5b40a7f700 1 Processor -- start 2023-03-27T10:17:19.868 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.866+0000 7f5b40a7f700 1 -- start start 2023-03-27T10:17:19.868 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.866+0000 7f5b40a7f700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f5b3c105810 0x7f5b3c194ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:19.868 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.866+0000 7f5b40a7f700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5b3c195420 0x7f5b3c199890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:19.868 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.866+0000 7f5b40a7f700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f5b3c199dd0 0x7f5b3c19a270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:19.868 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.866+0000 7f5b40a7f700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b3c195af0 con 0x7f5b3c195420 2023-03-27T10:17:19.868 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.866+0000 7f5b40a7f700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b3c1a1b40 con 0x7f5b3c105810 2023-03-27T10:17:19.868 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.866+0000 7f5b40a7f700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b3c1a1c80 con 0x7f5b3c199dd0 2023-03-27T10:17:19.869 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.866+0000 7f5b39d9b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5b3c195420 0x7f5b3c199890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:19.869 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.866+0000 7f5b39d9b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5b3c195420 0x7f5b3c199890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44750/0 (socket says 172.21.15.159:44750) 2023-03-27T10:17:19.869 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.866+0000 7f5b39d9b700 1 -- 172.21.15.159:0/2000781993 learned_addr learned my addr 172.21.15.159:0/2000781993 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:17:19.869 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.866+0000 7f5b3a59c700 1 --2- 172.21.15.159:0/2000781993 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f5b3c105810 0x7f5b3c194ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:19.869 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.866+0000 7f5b3ad9d700 1 --2- 172.21.15.159:0/2000781993 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f5b3c199dd0 0x7f5b3c19a270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:19.869 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.867+0000 7f5b39d9b700 1 -- 172.21.15.159:0/2000781993 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f5b3c199dd0 msgr2=0x7f5b3c19a270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:19.869 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.867+0000 7f5b39d9b700 1 --2- 172.21.15.159:0/2000781993 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f5b3c199dd0 0x7f5b3c19a270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:19.869 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.867+0000 7f5b39d9b700 1 -- 172.21.15.159:0/2000781993 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f5b3c105810 msgr2=0x7f5b3c194ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:19.870 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.867+0000 7f5b39d9b700 1 --2- 172.21.15.159:0/2000781993 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f5b3c105810 0x7f5b3c194ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:19.870 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.867+0000 7f5b39d9b700 1 -- 172.21.15.159:0/2000781993 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5b34009d00 con 0x7f5b3c195420 2023-03-27T10:17:19.870 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.867+0000 7f5b3a59c700 1 --2- 172.21.15.159:0/2000781993 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f5b3c105810 0x7f5b3c194ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:17:19.870 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.867+0000 7f5b39d9b700 1 --2- 172.21.15.159:0/2000781993 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5b3c195420 0x7f5b3c199890 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f5b2400b7b0 tx=0x7f5b2400bb70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:17:19.870 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.867+0000 7f5b337fe700 1 -- 172.21.15.159:0/2000781993 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b24004230 con 0x7f5b3c195420 2023-03-27T10:17:19.870 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.867+0000 7f5b337fe700 1 -- 172.21.15.159:0/2000781993 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5b24004390 con 0x7f5b3c195420 2023-03-27T10:17:19.870 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.867+0000 7f5b40a7f700 1 -- 172.21.15.159:0/2000781993 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5b3c109c10 con 0x7f5b3c195420 2023-03-27T10:17:19.871 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.867+0000 7f5b337fe700 1 -- 172.21.15.159:0/2000781993 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b24011ad0 con 0x7f5b3c195420 2023-03-27T10:17:19.871 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.867+0000 7f5b40a7f700 1 -- 172.21.15.159:0/2000781993 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5b3c109a80 con 0x7f5b3c195420 2023-03-27T10:17:19.871 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.868+0000 7f5b337fe700 1 -- 172.21.15.159:0/2000781993 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 13) v1 ==== 43566+0+0 (secure 0 0 0) 0x7f5b24017020 con 0x7f5b3c195420 2023-03-27T10:17:19.871 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.868+0000 7f5b337fe700 1 --2- 172.21.15.159:0/2000781993 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f5b28036d50 0x7f5b28039200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:19.871 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.868+0000 7f5b337fe700 1 -- 172.21.15.159:0/2000781993 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f5b240481d0 con 0x7f5b3c195420 2023-03-27T10:17:19.871 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.868+0000 7f5b40a7f700 1 -- 172.21.15.159:0/2000781993 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5b3c109da0 con 0x7f5b3c195420 2023-03-27T10:17:19.871 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.868+0000 7f5b3a59c700 1 --2- 172.21.15.159:0/2000781993 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f5b28036d50 0x7f5b28039200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:19.872 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.869+0000 7f5b3a59c700 1 --2- 172.21.15.159:0/2000781993 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f5b28036d50 0x7f5b28039200 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f5b34009610 tx=0x7f5b3400b7f0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:17:19.872 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:19.873+0000 7f5b337fe700 1 -- 172.21.15.159:0/2000781993 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f5b2400f2e0 con 0x7f5b3c195420 2023-03-27T10:17:20.003 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: mon.b calling monitor election 2023-03-27T10:17:20.003 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: Updating smithi159:/etc/ceph/ceph.conf 2023-03-27T10:17:20.003 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: Updating smithi161:/etc/ceph/ceph.conf 2023-03-27T10:17:20.003 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: Updating smithi171:/etc/ceph/ceph.conf 2023-03-27T10:17:20.003 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: mon.b calling monitor election 2023-03-27T10:17:20.004 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: mon.c calling monitor election 2023-03-27T10:17:20.004 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: mon.a calling monitor election 2023-03-27T10:17:20.004 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2023-03-27T10:17:20.004 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: monmap e3: 3 mons at {a=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0],b=[v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0],c=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0]} removed_ranks: {} 2023-03-27T10:17:20.004 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: fsmap 2023-03-27T10:17:20.004 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: osdmap e4: 0 total, 0 up, 0 in 2023-03-27T10:17:20.004 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: mgrmap e13: x(active, since 52s) 2023-03-27T10:17:20.004 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2023-03-27T10:17:20.005 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: Cluster is now healthy 2023-03-27T10:17:20.005 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: overall HEALTH_OK 2023-03-27T10:17:20.005 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:20.005 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: Updating smithi161:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.conf 2023-03-27T10:17:20.005 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: Updating smithi171:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.conf 2023-03-27T10:17:20.005 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: Updating smithi159:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.conf 2023-03-27T10:17:20.005 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.006 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.006 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.006 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.006 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.006 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.006 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.006 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: Reconfiguring mon.a (unknown last config time)... 2023-03-27T10:17:20.006 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-03-27T10:17:20.007 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-03-27T10:17:20.007 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:20.007 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:19 smithi159 ceph-mon[93039]: Reconfiguring daemon mon.a on smithi159 2023-03-27T10:17:20.071 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:20.072+0000 7f5b40a7f700 1 -- 172.21.15.159:0/2000781993 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "config generate-minimal-conf"} v 0) v1 -- 0x7f5b3c1a2720 con 0x7f5b3c195420 2023-03-27T10:17:20.071 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:20.072+0000 7f5b337fe700 1 -- 172.21.15.159:0/2000781993 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "config generate-minimal-conf"}]=0 v9) v1 ==== 76+0+277 (secure 0 0 0) 0x7f5b24045020 con 0x7f5b3c195420 2023-03-27T10:17:20.072 INFO:teuthology.orchestra.run.smithi159.stdout:# minimal ceph.conf for 2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:17:20.072 INFO:teuthology.orchestra.run.smithi159.stdout:[global] 2023-03-27T10:17:20.072 INFO:teuthology.orchestra.run.smithi159.stdout: fsid = 2c3cd140-cc88-11ed-9afe-001a4aab830c 2023-03-27T10:17:20.072 INFO:teuthology.orchestra.run.smithi159.stdout: mon_host = [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] 2023-03-27T10:17:20.073 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:20.074+0000 7f5b40a7f700 1 -- 172.21.15.159:0/2000781993 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f5b28036d50 msgr2=0x7f5b28039200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:20.074 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:20.074+0000 7f5b40a7f700 1 --2- 172.21.15.159:0/2000781993 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f5b28036d50 0x7f5b28039200 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f5b34009610 tx=0x7f5b3400b7f0 comp rx=0 tx=0).stop 2023-03-27T10:17:20.074 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:20.074+0000 7f5b40a7f700 1 -- 172.21.15.159:0/2000781993 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5b3c195420 msgr2=0x7f5b3c199890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:20.074 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:20.074+0000 7f5b40a7f700 1 --2- 172.21.15.159:0/2000781993 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5b3c195420 0x7f5b3c199890 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f5b2400b7b0 tx=0x7f5b2400bb70 comp rx=0 tx=0).stop 2023-03-27T10:17:20.074 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:20.074+0000 7f5b40a7f700 1 -- 172.21.15.159:0/2000781993 shutdown_connections 2023-03-27T10:17:20.074 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:20.074+0000 7f5b40a7f700 1 --2- 172.21.15.159:0/2000781993 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f5b28036d50 0x7f5b28039200 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:20.074 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:20.074+0000 7f5b40a7f700 1 --2- 172.21.15.159:0/2000781993 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f5b3c199dd0 0x7f5b3c19a270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:20.074 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:20.074+0000 7f5b40a7f700 1 --2- 172.21.15.159:0/2000781993 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f5b3c105810 0x7f5b3c194ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:20.075 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:20.074+0000 7f5b40a7f700 1 --2- 172.21.15.159:0/2000781993 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5b3c195420 0x7f5b3c199890 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:20.075 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:20.074+0000 7f5b40a7f700 1 -- 172.21.15.159:0/2000781993 >> 172.21.15.159:0/2000781993 conn(0x7f5b3c0fb330 msgr2=0x7f5b3c0fb700 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:17:20.075 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:20.074+0000 7f5b40a7f700 1 -- 172.21.15.159:0/2000781993 shutdown_connections 2023-03-27T10:17:20.075 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:20.074+0000 7f5b40a7f700 1 -- 172.21.15.159:0/2000781993 wait complete. 2023-03-27T10:17:20.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: mon.b calling monitor election 2023-03-27T10:17:20.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: Updating smithi159:/etc/ceph/ceph.conf 2023-03-27T10:17:20.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: Updating smithi161:/etc/ceph/ceph.conf 2023-03-27T10:17:20.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: Updating smithi171:/etc/ceph/ceph.conf 2023-03-27T10:17:20.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: mon.b calling monitor election 2023-03-27T10:17:20.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: mon.c calling monitor election 2023-03-27T10:17:20.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: mon.a calling monitor election 2023-03-27T10:17:20.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2023-03-27T10:17:20.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: monmap e3: 3 mons at {a=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0],b=[v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0],c=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0]} removed_ranks: {} 2023-03-27T10:17:20.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: fsmap 2023-03-27T10:17:20.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: osdmap e4: 0 total, 0 up, 0 in 2023-03-27T10:17:20.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: mgrmap e13: x(active, since 52s) 2023-03-27T10:17:20.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2023-03-27T10:17:20.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: Cluster is now healthy 2023-03-27T10:17:20.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: overall HEALTH_OK 2023-03-27T10:17:20.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:20.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: Updating smithi161:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.conf 2023-03-27T10:17:20.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: Updating smithi171:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.conf 2023-03-27T10:17:20.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: Updating smithi159:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.conf 2023-03-27T10:17:20.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: Reconfiguring mon.a (unknown last config time)... 2023-03-27T10:17:20.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-03-27T10:17:20.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-03-27T10:17:20.088 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:20.088 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:19 smithi161 ceph-mon[94333]: Reconfiguring daemon mon.a on smithi159 2023-03-27T10:17:20.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: mon.b calling monitor election 2023-03-27T10:17:20.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: Updating smithi159:/etc/ceph/ceph.conf 2023-03-27T10:17:20.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: Updating smithi161:/etc/ceph/ceph.conf 2023-03-27T10:17:20.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: Updating smithi171:/etc/ceph/ceph.conf 2023-03-27T10:17:20.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: mon.b calling monitor election 2023-03-27T10:17:20.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: mon.c calling monitor election 2023-03-27T10:17:20.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: mon.a calling monitor election 2023-03-27T10:17:20.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2023-03-27T10:17:20.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: monmap e3: 3 mons at {a=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0],b=[v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0],c=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0]} removed_ranks: {} 2023-03-27T10:17:20.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: fsmap 2023-03-27T10:17:20.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: osdmap e4: 0 total, 0 up, 0 in 2023-03-27T10:17:20.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: mgrmap e13: x(active, since 52s) 2023-03-27T10:17:20.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2023-03-27T10:17:20.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: Cluster is now healthy 2023-03-27T10:17:20.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: overall HEALTH_OK 2023-03-27T10:17:20.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:20.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: Updating smithi161:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.conf 2023-03-27T10:17:20.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: Updating smithi171:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.conf 2023-03-27T10:17:20.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: Updating smithi159:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/config/ceph.conf 2023-03-27T10:17:20.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:20.230 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: Reconfiguring mon.a (unknown last config time)... 2023-03-27T10:17:20.230 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-03-27T10:17:20.230 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-03-27T10:17:20.230 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:20.230 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:19 smithi171 ceph-mon[94808]: Reconfiguring daemon mon.a on smithi159 2023-03-27T10:17:20.701 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-03-27T10:17:20.702 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2023-03-27T10:17:20.702 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/etc/ceph/ceph.conf 2023-03-27T10:17:20.732 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2023-03-27T10:17:20.732 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-03-27T10:17:20.802 DEBUG:teuthology.orchestra.run.smithi161:> set -ex 2023-03-27T10:17:20.803 DEBUG:teuthology.orchestra.run.smithi161:> sudo dd of=/etc/ceph/ceph.conf 2023-03-27T10:17:20.839 DEBUG:teuthology.orchestra.run.smithi161:> set -ex 2023-03-27T10:17:20.839 DEBUG:teuthology.orchestra.run.smithi161:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-03-27T10:17:20.911 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2023-03-27T10:17:20.911 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/ceph/ceph.conf 2023-03-27T10:17:20.946 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2023-03-27T10:17:20.947 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-03-27T10:17:21.013 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:20 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:21.014 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:20 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/2000781993' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:21.018 INFO:tasks.cephadm:Adding mgr.x on smithi159 2023-03-27T10:17:21.018 INFO:tasks.cephadm:Adding mgr.y on smithi161 2023-03-27T10:17:21.018 INFO:tasks.cephadm:Adding mgr.z on smithi171 2023-03-27T10:17:21.018 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch apply mgr '3;smithi159=x;smithi161=y;smithi171=z' 2023-03-27T10:17:21.083 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:20 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:21.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:20 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/2000781993' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:21.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:20 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:17:21.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:20 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/2000781993' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:21.693 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.c/config 2023-03-27T10:17:22.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:21 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:22.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:21 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:22.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:21 smithi161 ceph-mon[94333]: Reconfiguring mon.b (monmap changed)... 2023-03-27T10:17:22.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:21 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-03-27T10:17:22.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:21 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-03-27T10:17:22.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:21 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:22.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:21 smithi161 ceph-mon[94333]: Reconfiguring daemon mon.b on smithi161 2023-03-27T10:17:22.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:21 smithi161 ceph-mon[94333]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:22.106 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:21 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:22.106 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:21 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:22.106 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:21 smithi171 ceph-mon[94808]: Reconfiguring mon.b (monmap changed)... 2023-03-27T10:17:22.106 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:21 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-03-27T10:17:22.106 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:21 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-03-27T10:17:22.107 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:21 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:22.107 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:21 smithi171 ceph-mon[94808]: Reconfiguring daemon mon.b on smithi161 2023-03-27T10:17:22.107 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:21 smithi171 ceph-mon[94808]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:22.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:21 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:22.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:21 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:22.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:21 smithi159 ceph-mon[93039]: Reconfiguring mon.b (monmap changed)... 2023-03-27T10:17:22.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:21 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-03-27T10:17:22.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:21 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-03-27T10:17:22.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:21 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:22.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:21 smithi159 ceph-mon[93039]: Reconfiguring daemon mon.b on smithi161 2023-03-27T10:17:22.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:21 smithi159 ceph-mon[93039]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:23.053 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.053+0000 7f691f257700 1 -- 172.21.15.171:0/2721736868 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f69180ff920 msgr2=0x7f69180ffd30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:23.054 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.053+0000 7f691f257700 1 --2- 172.21.15.171:0/2721736868 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f69180ff920 0x7f69180ffd30 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f690c009910 tx=0x7f690c009c20 comp rx=0 tx=0).stop 2023-03-27T10:17:23.054 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.053+0000 7f691f257700 1 -- 172.21.15.171:0/2721736868 shutdown_connections 2023-03-27T10:17:23.054 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.053+0000 7f691f257700 1 --2- 172.21.15.171:0/2721736868 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f69180ff920 0x7f69180ffd30 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:23.054 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.053+0000 7f691f257700 1 -- 172.21.15.171:0/2721736868 >> 172.21.15.171:0/2721736868 conn(0x7f69180faef0 msgr2=0x7f69180fd300 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:17:23.054 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.053+0000 7f691f257700 1 -- 172.21.15.171:0/2721736868 shutdown_connections 2023-03-27T10:17:23.054 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.053+0000 7f691f257700 1 -- 172.21.15.171:0/2721736868 wait complete. 2023-03-27T10:17:23.056 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.054+0000 7f691f257700 1 Processor -- start 2023-03-27T10:17:23.056 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.054+0000 7f691f257700 1 -- start start 2023-03-27T10:17:23.056 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.054+0000 7f691f257700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f69180ff920 0x7f6918074570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:23.057 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.054+0000 7f691f257700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6918072bc0 0x7f6918073030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:23.057 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.054+0000 7f691f257700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f6918073570 0x7f691819ec40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:23.057 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.054+0000 7f691f257700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f691819f180 con 0x7f69180ff920 2023-03-27T10:17:23.057 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.054+0000 7f691f257700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f691819f2c0 con 0x7f6918073570 2023-03-27T10:17:23.057 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.054+0000 7f691f257700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f691819f400 con 0x7f6918072bc0 2023-03-27T10:17:23.057 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.054+0000 7f691cff3700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f69180ff920 0x7f6918074570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:23.057 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.054+0000 7f6917fff700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6918072bc0 0x7f6918073030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:23.058 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.054+0000 7f6917fff700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6918072bc0 0x7f6918073030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.171:3300/0 says I am v2:172.21.15.171:55098/0 (socket says 172.21.15.171:55098) 2023-03-27T10:17:23.058 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.054+0000 7f6917fff700 1 -- 172.21.15.171:0/2260912313 learned_addr learned my addr 172.21.15.171:0/2260912313 (peer_addr_for_me v2:172.21.15.171:0/0) 2023-03-27T10:17:23.058 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.055+0000 7f691d7f4700 1 --2- 172.21.15.171:0/2260912313 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f6918073570 0x7f691819ec40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:23.058 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.055+0000 7f6917fff700 1 -- 172.21.15.171:0/2260912313 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f6918073570 msgr2=0x7f691819ec40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:23.058 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.055+0000 7f6917fff700 1 --2- 172.21.15.171:0/2260912313 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f6918073570 0x7f691819ec40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:23.058 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.055+0000 7f6917fff700 1 -- 172.21.15.171:0/2260912313 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f69180ff920 msgr2=0x7f6918074570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:23.058 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.055+0000 7f6917fff700 1 --2- 172.21.15.171:0/2260912313 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f69180ff920 0x7f6918074570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:23.059 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.055+0000 7f6917fff700 1 -- 172.21.15.171:0/2260912313 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f690c0095f0 con 0x7f6918072bc0 2023-03-27T10:17:23.059 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.055+0000 7f691cff3700 1 --2- 172.21.15.171:0/2260912313 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f69180ff920 0x7f6918074570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-03-27T10:17:23.059 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.055+0000 7f6917fff700 1 --2- 172.21.15.171:0/2260912313 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6918072bc0 0x7f6918073030 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f690400b780 tx=0x7f690400ba90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:17:23.059 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.055+0000 7f6915ffb700 1 -- 172.21.15.171:0/2260912313 <== mon.1 v2:172.21.15.171:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6904004230 con 0x7f6918072bc0 2023-03-27T10:17:23.059 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.055+0000 7f6915ffb700 1 -- 172.21.15.171:0/2260912313 <== mon.1 v2:172.21.15.171:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6904004390 con 0x7f6918072bc0 2023-03-27T10:17:23.059 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.055+0000 7f6915ffb700 1 -- 172.21.15.171:0/2260912313 <== mon.1 v2:172.21.15.171:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6904011ad0 con 0x7f6918072bc0 2023-03-27T10:17:23.059 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.055+0000 7f691f257700 1 -- 172.21.15.171:0/2260912313 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f69181057f0 con 0x7f6918072bc0 2023-03-27T10:17:23.059 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.055+0000 7f691f257700 1 -- 172.21.15.171:0/2260912313 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6918105660 con 0x7f6918072bc0 2023-03-27T10:17:23.060 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.056+0000 7f6915ffb700 1 -- 172.21.15.171:0/2260912313 <== mon.1 v2:172.21.15.171:3300/0 4 ==== mgrmap(e 13) v1 ==== 43566+0+0 (secure 0 0 0) 0x7f6904017020 con 0x7f6918072bc0 2023-03-27T10:17:23.060 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.056+0000 7f6915ffb700 1 --2- 172.21.15.171:0/2260912313 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6908036da0 0x7f6908039250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:23.060 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.056+0000 7f6915ffb700 1 -- 172.21.15.171:0/2260912313 <== mon.1 v2:172.21.15.171:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f69040480d0 con 0x7f6918072bc0 2023-03-27T10:17:23.060 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.056+0000 7f691f257700 1 -- 172.21.15.171:0/2260912313 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f691818e450 con 0x7f6918072bc0 2023-03-27T10:17:23.060 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.056+0000 7f691cff3700 1 --2- 172.21.15.171:0/2260912313 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6908036da0 0x7f6908039250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:23.060 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.057+0000 7f691cff3700 1 --2- 172.21.15.171:0/2260912313 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6908036da0 0x7f6908039250 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f690c006010 tx=0x7f690c0058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:17:23.061 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:23.062 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:23.062 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:23 smithi171 ceph-mon[94808]: Reconfiguring mon.c (monmap changed)... 2023-03-27T10:17:23.062 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-03-27T10:17:23.062 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-03-27T10:17:23.062 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:23.062 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:23 smithi171 ceph-mon[94808]: Reconfiguring daemon mon.c on smithi171 2023-03-27T10:17:23.063 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.061+0000 7f6915ffb700 1 -- 172.21.15.171:0/2260912313 <== mon.1 v2:172.21.15.171:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f6904004500 con 0x7f6918072bc0 2023-03-27T10:17:23.262 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.263+0000 7f691f257700 1 -- 172.21.15.171:0/2260912313 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi159=x;smithi161=y;smithi171=z", "target": ["mon-mgr", ""]}) v1 -- 0x7f6918061850 con 0x7f6908036da0 2023-03-27T10:17:23.269 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.269+0000 7f6915ffb700 1 -- 172.21.15.171:0/2260912313 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7f6918061850 con 0x7f6908036da0 2023-03-27T10:17:23.269 INFO:teuthology.orchestra.run.smithi171.stdout:Scheduled mgr update... 2023-03-27T10:17:23.271 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.271+0000 7f691f257700 1 -- 172.21.15.171:0/2260912313 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6908036da0 msgr2=0x7f6908039250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:23.271 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.271+0000 7f691f257700 1 --2- 172.21.15.171:0/2260912313 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6908036da0 0x7f6908039250 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f690c006010 tx=0x7f690c0058e0 comp rx=0 tx=0).stop 2023-03-27T10:17:23.272 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.271+0000 7f691f257700 1 -- 172.21.15.171:0/2260912313 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6918072bc0 msgr2=0x7f6918073030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:23.272 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.271+0000 7f691f257700 1 --2- 172.21.15.171:0/2260912313 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6918072bc0 0x7f6918073030 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f690400b780 tx=0x7f690400ba90 comp rx=0 tx=0).stop 2023-03-27T10:17:23.272 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.271+0000 7f691f257700 1 -- 172.21.15.171:0/2260912313 shutdown_connections 2023-03-27T10:17:23.272 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.271+0000 7f691f257700 1 --2- 172.21.15.171:0/2260912313 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6908036da0 0x7f6908039250 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:23.272 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.271+0000 7f691f257700 1 --2- 172.21.15.171:0/2260912313 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f6918073570 0x7f691819ec40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:23.272 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.271+0000 7f691f257700 1 --2- 172.21.15.171:0/2260912313 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f69180ff920 0x7f6918074570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:23.272 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.271+0000 7f691f257700 1 --2- 172.21.15.171:0/2260912313 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6918072bc0 0x7f6918073030 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:23.273 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.271+0000 7f691f257700 1 -- 172.21.15.171:0/2260912313 >> 172.21.15.171:0/2260912313 conn(0x7f69180faef0 msgr2=0x7f69180fbbb0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:17:23.273 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.272+0000 7f691f257700 1 -- 172.21.15.171:0/2260912313 shutdown_connections 2023-03-27T10:17:23.273 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:17:23.272+0000 7f691f257700 1 -- 172.21.15.171:0/2260912313 wait complete. 2023-03-27T10:17:23.333 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:23.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:23.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:23 smithi161 ceph-mon[94333]: Reconfiguring mon.c (monmap changed)... 2023-03-27T10:17:23.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-03-27T10:17:23.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-03-27T10:17:23.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:23.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:23 smithi161 ceph-mon[94333]: Reconfiguring daemon mon.c on smithi171 2023-03-27T10:17:23.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:23.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:23.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:23 smithi159 ceph-mon[93039]: Reconfiguring mon.c (monmap changed)... 2023-03-27T10:17:23.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-03-27T10:17:23.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-03-27T10:17:23.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:23.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:23 smithi159 ceph-mon[93039]: Reconfiguring daemon mon.c on smithi171 2023-03-27T10:17:23.751 DEBUG:teuthology.orchestra.run.smithi161:mgr.y> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mgr.y.service 2023-03-27T10:17:23.754 DEBUG:teuthology.orchestra.run.smithi171:mgr.z> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mgr.z.service 2023-03-27T10:17:23.756 INFO:tasks.cephadm:Deploying OSDs... 2023-03-27T10:17:23.756 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2023-03-27T10:17:23.757 DEBUG:teuthology.orchestra.run.smithi159:> dd if=/scratch_devs of=/dev/stdout 2023-03-27T10:17:23.778 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-03-27T10:17:23.778 DEBUG:teuthology.orchestra.run.smithi159:> stat /dev/vg_nvme/lv_1 2023-03-27T10:17:23.786 INFO:journalctl@ceph.mgr.z.smithi171.stdout:-- Logs begin at Mon 2023-03-27 10:03:15 UTC. -- 2023-03-27T10:17:23.788 INFO:journalctl@ceph.mgr.y.smithi161.stdout:-- Logs begin at Mon 2023-03-27 10:02:59 UTC. -- 2023-03-27T10:17:23.838 INFO:teuthology.orchestra.run.smithi159.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-03-27T10:17:23.838 INFO:teuthology.orchestra.run.smithi159.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-03-27T10:17:23.839 INFO:teuthology.orchestra.run.smithi159.stdout:Device: 6h/6d Inode: 365251 Links: 1 2023-03-27T10:17:23.839 INFO:teuthology.orchestra.run.smithi159.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-03-27T10:17:23.839 INFO:teuthology.orchestra.run.smithi159.stdout:Context: system_u:object_r:device_t:s0 2023-03-27T10:17:23.839 INFO:teuthology.orchestra.run.smithi159.stdout:Access: 2023-03-27 10:16:36.568227575 +0000 2023-03-27T10:17:23.839 INFO:teuthology.orchestra.run.smithi159.stdout:Modify: 2023-03-27 10:11:05.047274385 +0000 2023-03-27T10:17:23.839 INFO:teuthology.orchestra.run.smithi159.stdout:Change: 2023-03-27 10:11:05.047274385 +0000 2023-03-27T10:17:23.839 INFO:teuthology.orchestra.run.smithi159.stdout: Birth: - 2023-03-27T10:17:23.840 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-03-27T10:17:23.907 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records in 2023-03-27T10:17:23.907 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records out 2023-03-27T10:17:23.907 INFO:teuthology.orchestra.run.smithi159.stderr:512 bytes copied, 0.000224769 s, 2.3 MB/s 2023-03-27T10:17:23.908 DEBUG:teuthology.orchestra.run.smithi159:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-03-27T10:17:23.968 DEBUG:teuthology.orchestra.run.smithi159:> stat /dev/vg_nvme/lv_2 2023-03-27T10:17:24.026 INFO:teuthology.orchestra.run.smithi159.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-03-27T10:17:24.026 INFO:teuthology.orchestra.run.smithi159.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-03-27T10:17:24.026 INFO:teuthology.orchestra.run.smithi159.stdout:Device: 6h/6d Inode: 368669 Links: 1 2023-03-27T10:17:24.026 INFO:teuthology.orchestra.run.smithi159.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-03-27T10:17:24.026 INFO:teuthology.orchestra.run.smithi159.stdout:Context: system_u:object_r:device_t:s0 2023-03-27T10:17:24.027 INFO:teuthology.orchestra.run.smithi159.stdout:Access: 2023-03-27 10:16:36.568227575 +0000 2023-03-27T10:17:24.027 INFO:teuthology.orchestra.run.smithi159.stdout:Modify: 2023-03-27 10:11:05.629262799 +0000 2023-03-27T10:17:24.027 INFO:teuthology.orchestra.run.smithi159.stdout:Change: 2023-03-27 10:11:05.629262799 +0000 2023-03-27T10:17:24.027 INFO:teuthology.orchestra.run.smithi159.stdout: Birth: - 2023-03-27T10:17:24.027 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-03-27T10:17:24.092 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records in 2023-03-27T10:17:24.092 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records out 2023-03-27T10:17:24.092 INFO:teuthology.orchestra.run.smithi159.stderr:512 bytes copied, 0.000230166 s, 2.2 MB/s 2023-03-27T10:17:24.093 DEBUG:teuthology.orchestra.run.smithi159:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-03-27T10:17:24.151 DEBUG:teuthology.orchestra.run.smithi159:> stat /dev/vg_nvme/lv_3 2023-03-27T10:17:24.208 INFO:teuthology.orchestra.run.smithi159.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-03-27T10:17:24.208 INFO:teuthology.orchestra.run.smithi159.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-03-27T10:17:24.208 INFO:teuthology.orchestra.run.smithi159.stdout:Device: 6h/6d Inode: 368740 Links: 1 2023-03-27T10:17:24.209 INFO:teuthology.orchestra.run.smithi159.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-03-27T10:17:24.209 INFO:teuthology.orchestra.run.smithi159.stdout:Context: system_u:object_r:device_t:s0 2023-03-27T10:17:24.209 INFO:teuthology.orchestra.run.smithi159.stdout:Access: 2023-03-27 10:16:36.568227575 +0000 2023-03-27T10:17:24.209 INFO:teuthology.orchestra.run.smithi159.stdout:Modify: 2023-03-27 10:11:06.151252407 +0000 2023-03-27T10:17:24.209 INFO:teuthology.orchestra.run.smithi159.stdout:Change: 2023-03-27 10:11:06.151252407 +0000 2023-03-27T10:17:24.209 INFO:teuthology.orchestra.run.smithi159.stdout: Birth: - 2023-03-27T10:17:24.210 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-03-27T10:17:24.274 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records in 2023-03-27T10:17:24.274 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records out 2023-03-27T10:17:24.274 INFO:teuthology.orchestra.run.smithi159.stderr:512 bytes copied, 0.000215116 s, 2.4 MB/s 2023-03-27T10:17:24.275 DEBUG:teuthology.orchestra.run.smithi159:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-03-27T10:17:24.332 DEBUG:teuthology.orchestra.run.smithi159:> stat /dev/vg_nvme/lv_4 2023-03-27T10:17:24.388 INFO:teuthology.orchestra.run.smithi159.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-03-27T10:17:24.389 INFO:teuthology.orchestra.run.smithi159.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-03-27T10:17:24.389 INFO:teuthology.orchestra.run.smithi159.stdout:Device: 6h/6d Inode: 368922 Links: 1 2023-03-27T10:17:24.389 INFO:teuthology.orchestra.run.smithi159.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-03-27T10:17:24.389 INFO:teuthology.orchestra.run.smithi159.stdout:Context: system_u:object_r:device_t:s0 2023-03-27T10:17:24.389 INFO:teuthology.orchestra.run.smithi159.stdout:Access: 2023-03-27 10:16:36.568227575 +0000 2023-03-27T10:17:24.389 INFO:teuthology.orchestra.run.smithi159.stdout:Modify: 2023-03-27 10:11:06.679241896 +0000 2023-03-27T10:17:24.389 INFO:teuthology.orchestra.run.smithi159.stdout:Change: 2023-03-27 10:11:06.679241896 +0000 2023-03-27T10:17:24.390 INFO:teuthology.orchestra.run.smithi159.stdout: Birth: - 2023-03-27T10:17:24.390 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-03-27T10:17:24.452 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records in 2023-03-27T10:17:24.453 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records out 2023-03-27T10:17:24.453 INFO:teuthology.orchestra.run.smithi159.stderr:512 bytes copied, 0.000209727 s, 2.4 MB/s 2023-03-27T10:17:24.454 DEBUG:teuthology.orchestra.run.smithi159:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-03-27T10:17:24.511 DEBUG:teuthology.orchestra.run.smithi161:> set -ex 2023-03-27T10:17:24.511 DEBUG:teuthology.orchestra.run.smithi161:> dd if=/scratch_devs of=/dev/stdout 2023-03-27T10:17:24.531 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-03-27T10:17:24.532 DEBUG:teuthology.orchestra.run.smithi161:> stat /dev/vg_nvme/lv_1 2023-03-27T10:17:24.540 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:24 smithi171 ceph-mon[94808]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:24.541 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:24 smithi171 ceph-mon[94808]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi159=x;smithi161=y;smithi171=z", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:17:24.541 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:24 smithi171 ceph-mon[94808]: Saving service mgr spec with placement smithi159=x;smithi161=y;smithi171=z;count:3 2023-03-27T10:17:24.541 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:24.541 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:24.541 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:24.541 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:24.541 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:24.542 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:24.542 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:24.542 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-03-27T10:17:24.542 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-03-27T10:17:24.542 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-03-27T10:17:24.542 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:24.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:24 smithi161 ceph-mon[94333]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:24.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:24 smithi161 ceph-mon[94333]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi159=x;smithi161=y;smithi171=z", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:17:24.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:24 smithi161 ceph-mon[94333]: Saving service mgr spec with placement smithi159=x;smithi161=y;smithi171=z;count:3 2023-03-27T10:17:24.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:24.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:24.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:24.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:24.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:24.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:24.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:24.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-03-27T10:17:24.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-03-27T10:17:24.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-03-27T10:17:24.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:24.592 INFO:teuthology.orchestra.run.smithi161.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-03-27T10:17:24.592 INFO:teuthology.orchestra.run.smithi161.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-03-27T10:17:24.592 INFO:teuthology.orchestra.run.smithi161.stdout:Device: 6h/6d Inode: 365611 Links: 1 2023-03-27T10:17:24.593 INFO:teuthology.orchestra.run.smithi161.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-03-27T10:17:24.593 INFO:teuthology.orchestra.run.smithi161.stdout:Context: system_u:object_r:device_t:s0 2023-03-27T10:17:24.593 INFO:teuthology.orchestra.run.smithi161.stdout:Access: 2023-03-27 10:17:11.309695434 +0000 2023-03-27T10:17:24.593 INFO:teuthology.orchestra.run.smithi161.stdout:Modify: 2023-03-27 10:11:23.531285607 +0000 2023-03-27T10:17:24.593 INFO:teuthology.orchestra.run.smithi161.stdout:Change: 2023-03-27 10:11:23.531285607 +0000 2023-03-27T10:17:24.593 INFO:teuthology.orchestra.run.smithi161.stdout: Birth: - 2023-03-27T10:17:24.593 DEBUG:teuthology.orchestra.run.smithi161:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-03-27T10:17:24.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:24 smithi159 ceph-mon[93039]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:24.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:24 smithi159 ceph-mon[93039]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi159=x;smithi161=y;smithi171=z", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:17:24.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:24 smithi159 ceph-mon[93039]: Saving service mgr spec with placement smithi159=x;smithi161=y;smithi171=z;count:3 2023-03-27T10:17:24.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:24.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:24.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:24.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:24.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:24.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:24.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:24.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-03-27T10:17:24.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-03-27T10:17:24.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-03-27T10:17:24.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:24.663 INFO:teuthology.orchestra.run.smithi161.stderr:1+0 records in 2023-03-27T10:17:24.663 INFO:teuthology.orchestra.run.smithi161.stderr:1+0 records out 2023-03-27T10:17:24.663 INFO:teuthology.orchestra.run.smithi161.stderr:512 bytes copied, 0.000259535 s, 2.0 MB/s 2023-03-27T10:17:24.664 DEBUG:teuthology.orchestra.run.smithi161:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-03-27T10:17:24.724 DEBUG:teuthology.orchestra.run.smithi161:> stat /dev/vg_nvme/lv_2 2023-03-27T10:17:24.782 INFO:teuthology.orchestra.run.smithi161.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-03-27T10:17:24.782 INFO:teuthology.orchestra.run.smithi161.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-03-27T10:17:24.782 INFO:teuthology.orchestra.run.smithi161.stdout:Device: 6h/6d Inode: 365828 Links: 1 2023-03-27T10:17:24.783 INFO:teuthology.orchestra.run.smithi161.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-03-27T10:17:24.783 INFO:teuthology.orchestra.run.smithi161.stdout:Context: system_u:object_r:device_t:s0 2023-03-27T10:17:24.783 INFO:teuthology.orchestra.run.smithi161.stdout:Access: 2023-03-27 10:17:11.309695434 +0000 2023-03-27T10:17:24.783 INFO:teuthology.orchestra.run.smithi161.stdout:Modify: 2023-03-27 10:11:24.058274050 +0000 2023-03-27T10:17:24.783 INFO:teuthology.orchestra.run.smithi161.stdout:Change: 2023-03-27 10:11:24.058274050 +0000 2023-03-27T10:17:24.783 INFO:teuthology.orchestra.run.smithi161.stdout: Birth: - 2023-03-27T10:17:24.783 DEBUG:teuthology.orchestra.run.smithi161:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-03-27T10:17:24.850 INFO:teuthology.orchestra.run.smithi161.stderr:1+0 records in 2023-03-27T10:17:24.850 INFO:teuthology.orchestra.run.smithi161.stderr:1+0 records out 2023-03-27T10:17:24.851 INFO:teuthology.orchestra.run.smithi161.stderr:512 bytes copied, 0.000225165 s, 2.3 MB/s 2023-03-27T10:17:24.851 DEBUG:teuthology.orchestra.run.smithi161:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-03-27T10:17:24.910 DEBUG:teuthology.orchestra.run.smithi161:> stat /dev/vg_nvme/lv_3 2023-03-27T10:17:24.968 INFO:teuthology.orchestra.run.smithi161.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-03-27T10:17:24.968 INFO:teuthology.orchestra.run.smithi161.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-03-27T10:17:24.968 INFO:teuthology.orchestra.run.smithi161.stdout:Device: 6h/6d Inode: 361448 Links: 1 2023-03-27T10:17:24.968 INFO:teuthology.orchestra.run.smithi161.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-03-27T10:17:24.969 INFO:teuthology.orchestra.run.smithi161.stdout:Context: system_u:object_r:device_t:s0 2023-03-27T10:17:24.969 INFO:teuthology.orchestra.run.smithi161.stdout:Access: 2023-03-27 10:17:11.309695434 +0000 2023-03-27T10:17:24.969 INFO:teuthology.orchestra.run.smithi161.stdout:Modify: 2023-03-27 10:11:24.578262646 +0000 2023-03-27T10:17:24.969 INFO:teuthology.orchestra.run.smithi161.stdout:Change: 2023-03-27 10:11:24.578262646 +0000 2023-03-27T10:17:24.969 INFO:teuthology.orchestra.run.smithi161.stdout: Birth: - 2023-03-27T10:17:24.969 DEBUG:teuthology.orchestra.run.smithi161:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-03-27T10:17:25.035 INFO:teuthology.orchestra.run.smithi161.stderr:1+0 records in 2023-03-27T10:17:25.035 INFO:teuthology.orchestra.run.smithi161.stderr:1+0 records out 2023-03-27T10:17:25.035 INFO:teuthology.orchestra.run.smithi161.stderr:512 bytes copied, 0.000247245 s, 2.1 MB/s 2023-03-27T10:17:25.036 DEBUG:teuthology.orchestra.run.smithi161:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-03-27T10:17:25.094 DEBUG:teuthology.orchestra.run.smithi161:> stat /dev/vg_nvme/lv_4 2023-03-27T10:17:25.151 INFO:teuthology.orchestra.run.smithi161.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-03-27T10:17:25.151 INFO:teuthology.orchestra.run.smithi161.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-03-27T10:17:25.151 INFO:teuthology.orchestra.run.smithi161.stdout:Device: 6h/6d Inode: 365973 Links: 1 2023-03-27T10:17:25.151 INFO:teuthology.orchestra.run.smithi161.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-03-27T10:17:25.151 INFO:teuthology.orchestra.run.smithi161.stdout:Context: system_u:object_r:device_t:s0 2023-03-27T10:17:25.151 INFO:teuthology.orchestra.run.smithi161.stdout:Access: 2023-03-27 10:17:11.309695434 +0000 2023-03-27T10:17:25.152 INFO:teuthology.orchestra.run.smithi161.stdout:Modify: 2023-03-27 10:11:25.113250914 +0000 2023-03-27T10:17:25.152 INFO:teuthology.orchestra.run.smithi161.stdout:Change: 2023-03-27 10:11:25.113250914 +0000 2023-03-27T10:17:25.152 INFO:teuthology.orchestra.run.smithi161.stdout: Birth: - 2023-03-27T10:17:25.152 DEBUG:teuthology.orchestra.run.smithi161:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-03-27T10:17:25.216 INFO:teuthology.orchestra.run.smithi161.stderr:1+0 records in 2023-03-27T10:17:25.216 INFO:teuthology.orchestra.run.smithi161.stderr:1+0 records out 2023-03-27T10:17:25.216 INFO:teuthology.orchestra.run.smithi161.stderr:512 bytes copied, 0.000195754 s, 2.6 MB/s 2023-03-27T10:17:25.217 DEBUG:teuthology.orchestra.run.smithi161:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-03-27T10:17:25.274 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2023-03-27T10:17:25.274 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/scratch_devs of=/dev/stdout 2023-03-27T10:17:25.292 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-03-27T10:17:25.292 DEBUG:teuthology.orchestra.run.smithi171:> stat /dev/vg_nvme/lv_1 2023-03-27T10:17:25.333 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:25 smithi161 ceph-mon[94333]: Deploying daemon mgr.z on smithi171 2023-03-27T10:17:25.350 INFO:teuthology.orchestra.run.smithi171.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-03-27T10:17:25.350 INFO:teuthology.orchestra.run.smithi171.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-03-27T10:17:25.350 INFO:teuthology.orchestra.run.smithi171.stdout:Device: 6h/6d Inode: 361465 Links: 1 2023-03-27T10:17:25.350 INFO:teuthology.orchestra.run.smithi171.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-03-27T10:17:25.351 INFO:teuthology.orchestra.run.smithi171.stdout:Context: system_u:object_r:device_t:s0 2023-03-27T10:17:25.351 INFO:teuthology.orchestra.run.smithi171.stdout:Access: 2023-03-27 10:17:25.303904276 +0000 2023-03-27T10:17:25.351 INFO:teuthology.orchestra.run.smithi171.stdout:Modify: 2023-03-27 10:11:10.020111704 +0000 2023-03-27T10:17:25.351 INFO:teuthology.orchestra.run.smithi171.stdout:Change: 2023-03-27 10:11:10.020111704 +0000 2023-03-27T10:17:25.351 INFO:teuthology.orchestra.run.smithi171.stdout: Birth: - 2023-03-27T10:17:25.351 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-03-27T10:17:25.419 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:25 smithi171 ceph-mon[94808]: Deploying daemon mgr.z on smithi171 2023-03-27T10:17:25.420 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:25 smithi171 systemd[1]: Starting Ceph mgr.z for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:17:25.423 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records in 2023-03-27T10:17:25.423 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records out 2023-03-27T10:17:25.423 INFO:teuthology.orchestra.run.smithi171.stderr:512 bytes copied, 0.000224539 s, 2.3 MB/s 2023-03-27T10:17:25.424 DEBUG:teuthology.orchestra.run.smithi171:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-03-27T10:17:25.482 DEBUG:teuthology.orchestra.run.smithi171:> stat /dev/vg_nvme/lv_2 2023-03-27T10:17:25.540 INFO:teuthology.orchestra.run.smithi171.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-03-27T10:17:25.541 INFO:teuthology.orchestra.run.smithi171.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-03-27T10:17:25.541 INFO:teuthology.orchestra.run.smithi171.stdout:Device: 6h/6d Inode: 358382 Links: 1 2023-03-27T10:17:25.541 INFO:teuthology.orchestra.run.smithi171.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-03-27T10:17:25.541 INFO:teuthology.orchestra.run.smithi171.stdout:Context: system_u:object_r:device_t:s0 2023-03-27T10:17:25.541 INFO:teuthology.orchestra.run.smithi171.stdout:Access: 2023-03-27 10:17:25.304904257 +0000 2023-03-27T10:17:25.541 INFO:teuthology.orchestra.run.smithi171.stdout:Modify: 2023-03-27 10:11:10.565100658 +0000 2023-03-27T10:17:25.542 INFO:teuthology.orchestra.run.smithi171.stdout:Change: 2023-03-27 10:11:10.565100658 +0000 2023-03-27T10:17:25.542 INFO:teuthology.orchestra.run.smithi171.stdout: Birth: - 2023-03-27T10:17:25.542 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-03-27T10:17:25.606 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records in 2023-03-27T10:17:25.606 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records out 2023-03-27T10:17:25.607 INFO:teuthology.orchestra.run.smithi171.stderr:512 bytes copied, 0.000230399 s, 2.2 MB/s 2023-03-27T10:17:25.607 DEBUG:teuthology.orchestra.run.smithi171:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-03-27T10:17:25.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:25 smithi159 ceph-mon[93039]: Deploying daemon mgr.z on smithi171 2023-03-27T10:17:25.665 DEBUG:teuthology.orchestra.run.smithi171:> stat /dev/vg_nvme/lv_3 2023-03-27T10:17:25.722 INFO:teuthology.orchestra.run.smithi171.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-03-27T10:17:25.722 INFO:teuthology.orchestra.run.smithi171.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-03-27T10:17:25.722 INFO:teuthology.orchestra.run.smithi171.stdout:Device: 6h/6d Inode: 365851 Links: 1 2023-03-27T10:17:25.722 INFO:teuthology.orchestra.run.smithi171.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-03-27T10:17:25.722 INFO:teuthology.orchestra.run.smithi171.stdout:Context: system_u:object_r:device_t:s0 2023-03-27T10:17:25.723 INFO:teuthology.orchestra.run.smithi171.stdout:Access: 2023-03-27 10:17:25.304904257 +0000 2023-03-27T10:17:25.723 INFO:teuthology.orchestra.run.smithi171.stdout:Modify: 2023-03-27 10:11:11.069090442 +0000 2023-03-27T10:17:25.723 INFO:teuthology.orchestra.run.smithi171.stdout:Change: 2023-03-27 10:11:11.069090442 +0000 2023-03-27T10:17:25.723 INFO:teuthology.orchestra.run.smithi171.stdout: Birth: - 2023-03-27T10:17:25.723 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-03-27T10:17:25.726 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:25 smithi171 podman[96727]: 2023-03-27T10:17:25.751 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records in 2023-03-27T10:17:25.751 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records out 2023-03-27T10:17:25.751 INFO:teuthology.orchestra.run.smithi171.stderr:512 bytes copied, 0.00020922 s, 2.4 MB/s 2023-03-27T10:17:25.752 DEBUG:teuthology.orchestra.run.smithi171:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-03-27T10:17:25.810 DEBUG:teuthology.orchestra.run.smithi171:> stat /dev/vg_nvme/lv_4 2023-03-27T10:17:25.866 INFO:teuthology.orchestra.run.smithi171.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-03-27T10:17:25.866 INFO:teuthology.orchestra.run.smithi171.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-03-27T10:17:25.866 INFO:teuthology.orchestra.run.smithi171.stdout:Device: 6h/6d Inode: 362920 Links: 1 2023-03-27T10:17:25.866 INFO:teuthology.orchestra.run.smithi171.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-03-27T10:17:25.867 INFO:teuthology.orchestra.run.smithi171.stdout:Context: system_u:object_r:device_t:s0 2023-03-27T10:17:25.867 INFO:teuthology.orchestra.run.smithi171.stdout:Access: 2023-03-27 10:17:25.304904257 +0000 2023-03-27T10:17:25.867 INFO:teuthology.orchestra.run.smithi171.stdout:Modify: 2023-03-27 10:11:11.580080084 +0000 2023-03-27T10:17:25.867 INFO:teuthology.orchestra.run.smithi171.stdout:Change: 2023-03-27 10:11:11.580080084 +0000 2023-03-27T10:17:25.867 INFO:teuthology.orchestra.run.smithi171.stdout: Birth: - 2023-03-27T10:17:25.867 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-03-27T10:17:25.932 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records in 2023-03-27T10:17:25.932 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records out 2023-03-27T10:17:25.932 INFO:teuthology.orchestra.run.smithi171.stderr:512 bytes copied, 0.000247304 s, 2.1 MB/s 2023-03-27T10:17:25.933 DEBUG:teuthology.orchestra.run.smithi171:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-03-27T10:17:25.991 INFO:tasks.cephadm:Deploying osd.0 on smithi159 with /dev/vg_nvme/lv_4... 2023-03-27T10:17:25.992 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-03-27T10:17:26.030 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:25 smithi171 bash[96727]: 07b85b41280c47fd54ff2cf6014e377af7f59f9712ee4004f094ca0f893a065f 2023-03-27T10:17:26.030 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:25 smithi171 ceph-mgr[96910]: -- 172.21.15.171:0/1023632484 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ad7ac24b60 con 0x55ad7abfb000 2023-03-27T10:17:26.030 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:25 smithi171 systemd[1]: Started Ceph mgr.z for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:17:26.282 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:26 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:26.030+0000 7f7e10362200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-03-27T10:17:26.283 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:26 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:26.106+0000 7f7e10362200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-03-27T10:17:26.283 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:26 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:26.192+0000 7f7e10362200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-03-27T10:17:26.283 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:26 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:26.282+0000 7f7e10362200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-03-27T10:17:26.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:26 smithi159 ceph-mon[93039]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:26.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:26 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:26.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:26 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:26.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:26 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-03-27T10:17:26.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:26 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-03-27T10:17:26.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:26 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-03-27T10:17:26.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:26 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:26.641 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:17:26.722 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:26 smithi161 ceph-mon[94333]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:26.722 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:26 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:26.722 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:26 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:26.722 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:26 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-03-27T10:17:26.722 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:26 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-03-27T10:17:26.722 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:26 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-03-27T10:17:26.723 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:26 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:26.726 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:26 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:26.449+0000 7f7e10362200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-03-27T10:17:26.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:26 smithi171 ceph-mon[94808]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:26.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:26 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:26.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:26 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:26.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:26 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-03-27T10:17:26.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:26 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-03-27T10:17:26.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:26 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-03-27T10:17:26.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:26 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:27.058 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:26 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:26.754+0000 7f7e10362200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-03-27T10:17:27.058 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:26 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:26.840+0000 7f7e10362200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-03-27T10:17:27.059 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:26 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:26.967+0000 7f7e10362200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-03-27T10:17:27.367 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:27 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:27.058+0000 7f7e10362200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-03-27T10:17:27.599 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:27 smithi161 ceph-mon[94333]: Deploying daemon mgr.y on smithi161 2023-03-27T10:17:27.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:27 smithi159 ceph-mon[93039]: Deploying daemon mgr.y on smithi161 2023-03-27T10:17:27.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:27 smithi171 ceph-mon[94808]: Deploying daemon mgr.y on smithi161 2023-03-27T10:17:27.726 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:27 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:27.477+0000 7f7e10362200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-03-27T10:17:27.868 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:27 smithi161 systemd[1]: Starting Ceph mgr.y for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:17:28.132 INFO:teuthology.orchestra.run.smithi159.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-03-27T10:17:28.132 INFO:teuthology.orchestra.run.smithi159.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-03-27T10:17:28.132 INFO:teuthology.orchestra.run.smithi159.stderr: stderr: 10+0 records in 2023-03-27T10:17:28.132 INFO:teuthology.orchestra.run.smithi159.stderr:10+0 records out 2023-03-27T10:17:28.133 INFO:teuthology.orchestra.run.smithi159.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110386 s, 950 MB/s 2023-03-27T10:17:28.133 INFO:teuthology.orchestra.run.smithi159.stderr:--> Zapping successful for: 2023-03-27T10:17:28.186 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:27 smithi161 podman[95771]: 2023-03-27T10:17:28.187 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:28 smithi161 bash[95771]: 46c47a12ae24fb644196f7bab77f0a3508997962e9b2d131f45362ff14702374 2023-03-27T10:17:28.187 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:28 smithi161 ceph-mgr[95811]: -- 172.21.15.161:0/1733539768 <== mon.2 v2:172.21.15.161:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560f4ad9eb60 con 0x560f4ad74c00 2023-03-27T10:17:28.371 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:28 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:28.045+0000 7f7e10362200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-03-27T10:17:28.371 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:28 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:28.213+0000 7f7e10362200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-03-27T10:17:28.484 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:28 smithi159 ceph-mon[93039]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:28.484 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:28.484 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:28.485 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:28.486 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch daemon add osd smithi159:vg_nvme/lv_4 2023-03-27T10:17:28.505 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:28 smithi161 ceph-mon[94333]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:28.505 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:28.505 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:28.505 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:28.506 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:28 smithi161 systemd[1]: Started Ceph mgr.y for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:17:28.506 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:28 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:28.234+0000 7feb47e50200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-03-27T10:17:28.506 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:28 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:28.316+0000 7feb47e50200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-03-27T10:17:28.506 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:28 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:28.408+0000 7feb47e50200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-03-27T10:17:28.506 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:28 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:28.505+0000 7feb47e50200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-03-27T10:17:28.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:28 smithi171 ceph-mon[94808]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:28.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:28.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:28.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:28.834 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:28 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:28.683+0000 7feb47e50200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-03-27T10:17:29.128 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:17:29.264 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:28 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:28.997+0000 7f7e10362200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-03-27T10:17:29.264 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:29 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:29.094+0000 7f7e10362200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-03-27T10:17:29.321 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:29 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:29.004+0000 7feb47e50200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-03-27T10:17:29.321 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:29 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:29.097+0000 7feb47e50200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-03-27T10:17:29.321 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:29 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:29.229+0000 7feb47e50200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-03-27T10:17:29.584 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:29 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:29.322+0000 7feb47e50200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-03-27T10:17:29.609 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:29 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:29.264+0000 7f7e10362200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-03-27T10:17:29.609 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:29 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:29.454+0000 7f7e10362200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-03-27T10:17:29.976 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:29 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:29.608+0000 7f7e10362200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-03-27T10:17:30.084 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:29 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:29.796+0000 7feb47e50200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-03-27T10:17:30.361 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.361+0000 7f6ced4d2700 1 -- 172.21.15.159:0/805233540 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6ce81031b0 msgr2=0x7f6ce8105590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:30.362 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.361+0000 7f6ced4d2700 1 --2- 172.21.15.159:0/805233540 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6ce81031b0 0x7f6ce8105590 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f6cdc009990 tx=0x7f6cdc009ca0 comp rx=0 tx=0).stop 2023-03-27T10:17:30.362 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.361+0000 7f6ced4d2700 1 -- 172.21.15.159:0/805233540 shutdown_connections 2023-03-27T10:17:30.362 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.361+0000 7f6ced4d2700 1 --2- 172.21.15.159:0/805233540 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6ce8105ad0 0x7f6ce8107ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:30.362 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.361+0000 7f6ced4d2700 1 --2- 172.21.15.159:0/805233540 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f6ce8100890 0x7f6ce8102c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:30.362 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.361+0000 7f6ced4d2700 1 --2- 172.21.15.159:0/805233540 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6ce81031b0 0x7f6ce8105590 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:30.362 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.361+0000 7f6ced4d2700 1 -- 172.21.15.159:0/805233540 >> 172.21.15.159:0/805233540 conn(0x7f6ce80fa2b0 msgr2=0x7f6ce80fc720 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:17:30.363 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.361+0000 7f6ced4d2700 1 -- 172.21.15.159:0/805233540 shutdown_connections 2023-03-27T10:17:30.363 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.361+0000 7f6ced4d2700 1 -- 172.21.15.159:0/805233540 wait complete. 2023-03-27T10:17:30.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.362+0000 7f6ced4d2700 1 Processor -- start 2023-03-27T10:17:30.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.363+0000 7f6ced4d2700 1 -- start start 2023-03-27T10:17:30.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.363+0000 7f6ced4d2700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f6ce8100890 0x7f6ce81001f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:30.365 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.363+0000 7f6ced4d2700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6ce81031b0 0x7f6ce80fe840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:30.365 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.363+0000 7f6ced4d2700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6ce8105ad0 0x7f6ce80fed80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:30.365 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.363+0000 7f6ced4d2700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ce80ff500 con 0x7f6ce8105ad0 2023-03-27T10:17:30.365 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.363+0000 7f6ced4d2700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ce80ff670 con 0x7f6ce8100890 2023-03-27T10:17:30.366 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.363+0000 7f6ced4d2700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ce819fb20 con 0x7f6ce81031b0 2023-03-27T10:17:30.366 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.363+0000 7f6ce77fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6ce8105ad0 0x7f6ce80fed80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:30.366 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.363+0000 7f6ce77fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6ce8105ad0 0x7f6ce80fed80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44762/0 (socket says 172.21.15.159:44762) 2023-03-27T10:17:30.366 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.363+0000 7f6ce77fe700 1 -- 172.21.15.159:0/1398972593 learned_addr learned my addr 172.21.15.159:0/1398972593 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:17:30.366 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.363+0000 7f6ce77fe700 1 -- 172.21.15.159:0/1398972593 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6ce81031b0 msgr2=0x7f6ce80fe840 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-03-27T10:17:30.366 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.363+0000 7f6ce77fe700 1 --2- 172.21.15.159:0/1398972593 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6ce81031b0 0x7f6ce80fe840 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:30.366 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.363+0000 7f6ce77fe700 1 -- 172.21.15.159:0/1398972593 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f6ce8100890 msgr2=0x7f6ce81001f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:30.367 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.363+0000 7f6ce77fe700 1 --2- 172.21.15.159:0/1398972593 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f6ce8100890 0x7f6ce81001f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:30.367 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.363+0000 7f6ce77fe700 1 -- 172.21.15.159:0/1398972593 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6cdc0095f0 con 0x7f6ce8105ad0 2023-03-27T10:17:30.367 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.364+0000 7f6ce77fe700 1 --2- 172.21.15.159:0/1398972593 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6ce8105ad0 0x7f6ce80fed80 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f6ce000ef90 tx=0x7f6ce000c450 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:17:30.367 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.365+0000 7f6ccffff700 1 -- 172.21.15.159:0/1398972593 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ce0018070 con 0x7f6ce8105ad0 2023-03-27T10:17:30.367 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.365+0000 7f6ccffff700 1 -- 172.21.15.159:0/1398972593 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6ce000f040 con 0x7f6ce8105ad0 2023-03-27T10:17:30.367 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.365+0000 7f6ced4d2700 1 -- 172.21.15.159:0/1398972593 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6ce810b720 con 0x7f6ce8105ad0 2023-03-27T10:17:30.367 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.365+0000 7f6ccffff700 1 -- 172.21.15.159:0/1398972593 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ce0003e40 con 0x7f6ce8105ad0 2023-03-27T10:17:30.368 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.365+0000 7f6ced4d2700 1 -- 172.21.15.159:0/1398972593 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6ce810b590 con 0x7f6ce8105ad0 2023-03-27T10:17:30.368 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.366+0000 7f6ccffff700 1 -- 172.21.15.159:0/1398972593 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 13) v1 ==== 43566+0+0 (secure 0 0 0) 0x7f6ce0004020 con 0x7f6ce8105ad0 2023-03-27T10:17:30.368 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.366+0000 7f6ccffff700 1 --2- 172.21.15.159:0/1398972593 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6cd0036a30 0x7f6cd0038ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:30.369 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.366+0000 7f6ccffff700 1 -- 172.21.15.159:0/1398972593 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f6ce0048580 con 0x7f6ce8105ad0 2023-03-27T10:17:30.369 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.366+0000 7f6ced4d2700 1 -- 172.21.15.159:0/1398972593 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6ce80fbe20 con 0x7f6ce8105ad0 2023-03-27T10:17:30.369 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.366+0000 7f6ce6ffd700 1 --2- 172.21.15.159:0/1398972593 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6cd0036a30 0x7f6cd0038ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:30.369 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.366+0000 7f6ce6ffd700 1 --2- 172.21.15.159:0/1398972593 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6cd0036a30 0x7f6cd0038ee0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f6cd8005d90 tx=0x7f6cd8005d00 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:17:30.370 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.371+0000 7f6ccffff700 1 -- 172.21.15.159:0/1398972593 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f6ce0007500 con 0x7f6ce8105ad0 2023-03-27T10:17:30.375 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:30 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:30.018+0000 7f7e10362200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-03-27T10:17:30.569 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:30.569+0000 7f6ced4d2700 1 -- 172.21.15.159:0/1398972593 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) v1 -- 0x7f6ce8061670 con 0x7f6cd0036a30 2023-03-27T10:17:30.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:30 smithi159 ceph-mon[93039]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:30.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:30.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:30.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:30.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:30.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:30.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:30.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:30.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-03-27T10:17:30.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-03-27T10:17:30.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:30.726 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:30 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:30.541+0000 7f7e10362200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-03-27T10:17:30.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:30 smithi171 ceph-mon[94808]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:30.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:30.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:30.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:30.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:30.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:30.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:30.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:30.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-03-27T10:17:30.728 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-03-27T10:17:30.728 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:30.834 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:30 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:30.386+0000 7feb47e50200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-03-27T10:17:30.834 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:30 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:30.554+0000 7feb47e50200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-03-27T10:17:30.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:30 smithi161 ceph-mon[94333]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:30.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:30.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:30.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:30.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:30.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:30.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:30.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:30.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-03-27T10:17:30.836 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-03-27T10:17:30.836 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:31.148 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:30 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:30.848+0000 7f7e10362200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-03-27T10:17:31.148 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:31 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:31.036+0000 7f7e10362200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-03-27T10:17:31.476 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:31 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:31.148+0000 7f7e10362200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-03-27T10:17:31.476 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:31 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:31.358+0000 7f7e10362200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-03-27T10:17:31.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:31 smithi171 ceph-mon[94808]: Reconfiguring mgr.x (unknown last config time)... 2023-03-27T10:17:31.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:31 smithi171 ceph-mon[94808]: Reconfiguring daemon mgr.x on smithi159 2023-03-27T10:17:31.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:31 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:17:31.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:31 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:17:31.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:31 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:31.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:31 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:31.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:31 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:31.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:31 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:31.558 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:31 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:31.303+0000 7feb47e50200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-03-27T10:17:31.558 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:31 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:31.393+0000 7feb47e50200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-03-27T10:17:31.559 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:31 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:31.559+0000 7feb47e50200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-03-27T10:17:31.559 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:31 smithi161 ceph-mon[94333]: Reconfiguring mgr.x (unknown last config time)... 2023-03-27T10:17:31.559 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:31 smithi161 ceph-mon[94333]: Reconfiguring daemon mgr.x on smithi159 2023-03-27T10:17:31.559 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:31 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:17:31.559 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:31 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:17:31.560 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:31 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:31.560 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:31 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:31.560 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:31 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:31.560 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:31 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:31.629 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:31 smithi159 ceph-mon[93039]: Reconfiguring mgr.x (unknown last config time)... 2023-03-27T10:17:31.629 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:31 smithi159 ceph-mon[93039]: Reconfiguring daemon mgr.x on smithi159 2023-03-27T10:17:31.629 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:31 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:17:31.629 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:31 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:17:31.629 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:31 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:31.630 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:31 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:31.630 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:31 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:31.630 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:31 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:31.834 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:31 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:31.741+0000 7feb47e50200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-03-27T10:17:32.282 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:31 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:31.886+0000 7feb47e50200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-03-27T10:17:32.476 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:17:32 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z[96805]: 2023-03-27T10:17:32.209+0000 7f7e10362200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-03-27T10:17:32.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:32 smithi171 ceph-mon[94808]: from='client.14226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:17:32.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:32 smithi171 ceph-mon[94808]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:32.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:32 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:32.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:32 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:32.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:32 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:32.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:32 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:32.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:32 smithi171 ceph-mon[94808]: Standby manager daemon z started 2023-03-27T10:17:32.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:32 smithi171 ceph-mon[94808]: from='mgr.? 172.21.15.171:0/3351152611' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2023-03-27T10:17:32.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:32 smithi171 ceph-mon[94808]: from='mgr.? 172.21.15.171:0/3351152611' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-03-27T10:17:32.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:32 smithi171 ceph-mon[94808]: from='mgr.? 172.21.15.171:0/3351152611' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2023-03-27T10:17:32.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:32 smithi171 ceph-mon[94808]: from='mgr.? 172.21.15.171:0/3351152611' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-03-27T10:17:32.584 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:32 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:32.283+0000 7feb47e50200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-03-27T10:17:32.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:32 smithi161 ceph-mon[94333]: from='client.14226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:17:32.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:32 smithi161 ceph-mon[94333]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:32.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:32 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:32.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:32 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:32.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:32 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:32.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:32 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:32.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:32 smithi161 ceph-mon[94333]: Standby manager daemon z started 2023-03-27T10:17:32.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:32 smithi161 ceph-mon[94333]: from='mgr.? 172.21.15.171:0/3351152611' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2023-03-27T10:17:32.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:32 smithi161 ceph-mon[94333]: from='mgr.? 172.21.15.171:0/3351152611' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-03-27T10:17:32.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:32 smithi161 ceph-mon[94333]: from='mgr.? 172.21.15.171:0/3351152611' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2023-03-27T10:17:32.586 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:32 smithi161 ceph-mon[94333]: from='mgr.? 172.21.15.171:0/3351152611' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-03-27T10:17:32.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:32 smithi159 ceph-mon[93039]: from='client.14226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:17:32.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:32 smithi159 ceph-mon[93039]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:32.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:32 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:32.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:32 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:32.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:32 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:32.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:32 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:32.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:32 smithi159 ceph-mon[93039]: Standby manager daemon z started 2023-03-27T10:17:32.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:32 smithi159 ceph-mon[93039]: from='mgr.? 172.21.15.171:0/3351152611' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2023-03-27T10:17:32.867 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:32 smithi159 ceph-mon[93039]: from='mgr.? 172.21.15.171:0/3351152611' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-03-27T10:17:32.867 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:32 smithi159 ceph-mon[93039]: from='mgr.? 172.21.15.171:0/3351152611' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2023-03-27T10:17:32.867 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:32 smithi159 ceph-mon[93039]: from='mgr.? 172.21.15.171:0/3351152611' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-03-27T10:17:33.015 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:33.015+0000 7f6ccffff700 1 -- 172.21.15.159:0/1398972593 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mgrmap(e 14) v1 ==== 86692+0+0 (secure 0 0 0) 0x7f6ce0004ad0 con 0x7f6ce8105ad0 2023-03-27T10:17:33.084 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:32 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:32.807+0000 7feb47e50200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-03-27T10:17:33.409 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:33 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:33.114+0000 7feb47e50200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-03-27T10:17:33.410 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:33 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:33.299+0000 7feb47e50200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-03-27T10:17:33.834 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:33 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:33.410+0000 7feb47e50200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-03-27T10:17:33.834 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:33 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:33.620+0000 7feb47e50200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-03-27T10:17:34.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:34 smithi161 ceph-mon[94333]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:34.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:34 smithi161 ceph-mon[94333]: mgrmap e14: x(active, since 66s), standbys: z 2023-03-27T10:17:34.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:34 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2023-03-27T10:17:34.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:34 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/2537742812' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f47f99b9-47f9-4d94-83d9-33b1278be17e"}]: dispatch 2023-03-27T10:17:34.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:34 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/2537742812' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f47f99b9-47f9-4d94-83d9-33b1278be17e"}]': finished 2023-03-27T10:17:34.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:34 smithi161 ceph-mon[94333]: osdmap e5: 1 total, 0 up, 1 in 2023-03-27T10:17:34.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:34 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-03-27T10:17:34.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:34 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/1869280959' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:17:34.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:34 smithi159 ceph-mon[93039]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:34.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:34 smithi159 ceph-mon[93039]: mgrmap e14: x(active, since 66s), standbys: z 2023-03-27T10:17:34.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:34 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2023-03-27T10:17:34.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:34 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/2537742812' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f47f99b9-47f9-4d94-83d9-33b1278be17e"}]: dispatch 2023-03-27T10:17:34.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:34 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/2537742812' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f47f99b9-47f9-4d94-83d9-33b1278be17e"}]': finished 2023-03-27T10:17:34.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:34 smithi159 ceph-mon[93039]: osdmap e5: 1 total, 0 up, 1 in 2023-03-27T10:17:34.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:34 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-03-27T10:17:34.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:34 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/1869280959' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:17:34.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:34 smithi171 ceph-mon[94808]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:34.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:34 smithi171 ceph-mon[94808]: mgrmap e14: x(active, since 66s), standbys: z 2023-03-27T10:17:34.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:34 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2023-03-27T10:17:34.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:34 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/2537742812' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f47f99b9-47f9-4d94-83d9-33b1278be17e"}]: dispatch 2023-03-27T10:17:34.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:34 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/2537742812' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f47f99b9-47f9-4d94-83d9-33b1278be17e"}]': finished 2023-03-27T10:17:34.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:34 smithi171 ceph-mon[94808]: osdmap e5: 1 total, 0 up, 1 in 2023-03-27T10:17:34.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:34 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-03-27T10:17:34.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:34 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/1869280959' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:17:34.834 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:17:34 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y[95787]: 2023-03-27T10:17:34.463+0000 7feb47e50200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-03-27T10:17:35.025 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:35.025+0000 7f6ccffff700 1 -- 172.21.15.159:0/1398972593 <== mon.0 v2:172.21.15.159:3300/0 8 ==== mgrmap(e 15) v1 ==== 129818+0+0 (secure 0 0 0) 0x7f6ce001c3c0 con 0x7f6ce8105ad0 2023-03-27T10:17:35.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:35 smithi161 ceph-mon[94333]: Standby manager daemon y started 2023-03-27T10:17:35.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:35 smithi161 ceph-mon[94333]: from='mgr.? 172.21.15.161:0/3048320161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-03-27T10:17:35.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:35 smithi161 ceph-mon[94333]: from='mgr.? 172.21.15.161:0/3048320161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-03-27T10:17:35.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:35 smithi161 ceph-mon[94333]: from='mgr.? 172.21.15.161:0/3048320161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-03-27T10:17:35.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:35 smithi161 ceph-mon[94333]: from='mgr.? 172.21.15.161:0/3048320161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-03-27T10:17:35.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:35 smithi159 ceph-mon[93039]: Standby manager daemon y started 2023-03-27T10:17:35.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:35 smithi159 ceph-mon[93039]: from='mgr.? 172.21.15.161:0/3048320161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-03-27T10:17:35.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:35 smithi159 ceph-mon[93039]: from='mgr.? 172.21.15.161:0/3048320161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-03-27T10:17:35.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:35 smithi159 ceph-mon[93039]: from='mgr.? 172.21.15.161:0/3048320161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-03-27T10:17:35.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:35 smithi159 ceph-mon[93039]: from='mgr.? 172.21.15.161:0/3048320161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-03-27T10:17:35.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:35 smithi171 ceph-mon[94808]: Standby manager daemon y started 2023-03-27T10:17:35.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:35 smithi171 ceph-mon[94808]: from='mgr.? 172.21.15.161:0/3048320161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-03-27T10:17:35.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:35 smithi171 ceph-mon[94808]: from='mgr.? 172.21.15.161:0/3048320161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-03-27T10:17:35.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:35 smithi171 ceph-mon[94808]: from='mgr.? 172.21.15.161:0/3048320161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-03-27T10:17:35.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:35 smithi171 ceph-mon[94808]: from='mgr.? 172.21.15.161:0/3048320161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-03-27T10:17:36.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:36 smithi161 ceph-mon[94333]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:36.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:36 smithi161 ceph-mon[94333]: mgrmap e15: x(active, since 68s), standbys: z, y 2023-03-27T10:17:36.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:36 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-03-27T10:17:36.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:36 smithi159 ceph-mon[93039]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:36.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:36 smithi159 ceph-mon[93039]: mgrmap e15: x(active, since 68s), standbys: z, y 2023-03-27T10:17:36.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-03-27T10:17:36.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:36 smithi171 ceph-mon[94808]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:36.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:36 smithi171 ceph-mon[94808]: mgrmap e15: x(active, since 68s), standbys: z, y 2023-03-27T10:17:36.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:36 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-03-27T10:17:38.304 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:38 smithi159 ceph-mon[93039]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:38.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:38 smithi161 ceph-mon[94333]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:38.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:38 smithi171 ceph-mon[94808]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:40.282 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:40 smithi159 ceph-mon[93039]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:40.283 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:40 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-03-27T10:17:40.283 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:40 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:40.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:40 smithi161 ceph-mon[94333]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:40.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:40 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-03-27T10:17:40.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:40 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:40.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:40 smithi171 ceph-mon[94808]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:40.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:40 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-03-27T10:17:40.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:40 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:41.583 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:41 smithi159 ceph-mon[93039]: Deploying daemon osd.0 on smithi159 2023-03-27T10:17:41.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:41 smithi171 ceph-mon[94808]: Deploying daemon osd.0 on smithi159 2023-03-27T10:17:41.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:41 smithi161 ceph-mon[94333]: Deploying daemon osd.0 on smithi159 2023-03-27T10:17:42.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:42 smithi159 ceph-mon[93039]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:42.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:42 smithi171 ceph-mon[94808]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:42.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:42 smithi161 ceph-mon[94333]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:44.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:44 smithi171 ceph-mon[94808]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:44.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:44.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:44.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:44.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:44 smithi161 ceph-mon[94333]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:44.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:44.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:44.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:44.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:44 smithi159 ceph-mon[93039]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:44.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:44.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:44.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:46.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:46 smithi171 ceph-mon[94808]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:46.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:46 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:46.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:46 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:46.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:46 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:46.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:46 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:46.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:46 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:46.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:46 smithi161 ceph-mon[94333]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:46.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:46 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:46.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:46 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:46.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:46 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:46.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:46 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:46.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:46 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:46.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:46 smithi159 ceph-mon[93039]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:46.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:46 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:46.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:46 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:46.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:46 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:46.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:46 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:46.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:46 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:46.920 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:46.920+0000 7f6ccffff700 1 -- 172.21.15.159:0/1398972593 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f6ce8061670 con 0x7f6cd0036a30 2023-03-27T10:17:46.920 INFO:teuthology.orchestra.run.smithi159.stdout:Created osd(s) 0 on host 'smithi159' 2023-03-27T10:17:46.922 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:46.923+0000 7f6ced4d2700 1 -- 172.21.15.159:0/1398972593 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6cd0036a30 msgr2=0x7f6cd0038ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:46.922 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:46.923+0000 7f6ced4d2700 1 --2- 172.21.15.159:0/1398972593 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6cd0036a30 0x7f6cd0038ee0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f6cd8005d90 tx=0x7f6cd8005d00 comp rx=0 tx=0).stop 2023-03-27T10:17:46.922 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:46.923+0000 7f6ced4d2700 1 -- 172.21.15.159:0/1398972593 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6ce8105ad0 msgr2=0x7f6ce80fed80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:46.923 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:46.923+0000 7f6ced4d2700 1 --2- 172.21.15.159:0/1398972593 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6ce8105ad0 0x7f6ce80fed80 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f6ce000ef90 tx=0x7f6ce000c450 comp rx=0 tx=0).stop 2023-03-27T10:17:46.923 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:46.923+0000 7f6ced4d2700 1 -- 172.21.15.159:0/1398972593 shutdown_connections 2023-03-27T10:17:46.923 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:46.923+0000 7f6ced4d2700 1 --2- 172.21.15.159:0/1398972593 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6cd0036a30 0x7f6cd0038ee0 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:46.923 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:46.923+0000 7f6ced4d2700 1 --2- 172.21.15.159:0/1398972593 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6ce8105ad0 0x7f6ce80fed80 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:46.923 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:46.923+0000 7f6ced4d2700 1 --2- 172.21.15.159:0/1398972593 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f6ce8100890 0x7f6ce81001f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:46.923 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:46.923+0000 7f6ced4d2700 1 --2- 172.21.15.159:0/1398972593 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6ce81031b0 0x7f6ce80fe840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:46.923 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:46.923+0000 7f6ced4d2700 1 -- 172.21.15.159:0/1398972593 >> 172.21.15.159:0/1398972593 conn(0x7f6ce80fa2b0 msgr2=0x7f6ce8101550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:17:46.923 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:46.923+0000 7f6ced4d2700 1 -- 172.21.15.159:0/1398972593 shutdown_connections 2023-03-27T10:17:46.924 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:17:46.923+0000 7f6ced4d2700 1 -- 172.21.15.159:0/1398972593 wait complete. 2023-03-27T10:17:47.339 DEBUG:teuthology.orchestra.run.smithi159:osd.0> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.0.service 2023-03-27T10:17:47.341 INFO:tasks.cephadm:Deploying osd.1 on smithi161 with /dev/vg_nvme/lv_4... 2023-03-27T10:17:47.341 DEBUG:teuthology.orchestra.run.smithi161:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-03-27T10:17:47.379 INFO:journalctl@ceph.osd.0.smithi159.stdout:-- Logs begin at Mon 2023-03-27 10:03:12 UTC. -- 2023-03-27T10:17:47.987 INFO:teuthology.orchestra.run.smithi161.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.b/config 2023-03-27T10:17:48.089 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:47 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:48.089 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:47 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:48.089 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:47 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:48.089 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:47 smithi159 ceph-mon[93039]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:48.219 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:47 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:48.219 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:47 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:48.220 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:47 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:48.220 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:47 smithi161 ceph-mon[94333]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:48.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:47 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:48.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:47 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:48.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:47 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:17:48.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:47 smithi171 ceph-mon[94808]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:48.364 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 27 10:17:48 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-0[106031]: 2023-03-27T10:17:48.088+0000 7f1177c47580 -1 osd.0 0 log_to_monitors true 2023-03-27T10:17:49.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:48 smithi159 ceph-mon[93039]: from='osd.0 [v2:172.21.15.159:6802/87273908,v1:172.21.15.159:6803/87273908]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-03-27T10:17:49.153 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:48 smithi161 ceph-mon[94333]: from='osd.0 [v2:172.21.15.159:6802/87273908,v1:172.21.15.159:6803/87273908]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-03-27T10:17:49.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:48 smithi171 ceph-mon[94808]: from='osd.0 [v2:172.21.15.159:6802/87273908,v1:172.21.15.159:6803/87273908]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-03-27T10:17:49.687 INFO:teuthology.orchestra.run.smithi161.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-03-27T10:17:49.687 INFO:teuthology.orchestra.run.smithi161.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-03-27T10:17:49.687 INFO:teuthology.orchestra.run.smithi161.stderr: stderr: 10+0 records in 2023-03-27T10:17:49.687 INFO:teuthology.orchestra.run.smithi161.stderr:10+0 records out 2023-03-27T10:17:49.687 INFO:teuthology.orchestra.run.smithi161.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0111074 s, 944 MB/s 2023-03-27T10:17:49.688 INFO:teuthology.orchestra.run.smithi161.stderr: stderr: 2023-03-27T10:17:49.688 INFO:teuthology.orchestra.run.smithi161.stderr:--> Zapping successful for: 2023-03-27T10:17:50.083 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:49 smithi161 ceph-mon[94333]: from='osd.0 [v2:172.21.15.159:6802/87273908,v1:172.21.15.159:6803/87273908]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-03-27T10:17:50.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:49 smithi161 ceph-mon[94333]: osdmap e6: 1 total, 0 up, 1 in 2023-03-27T10:17:50.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-03-27T10:17:50.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:49 smithi161 ceph-mon[94333]: from='osd.0 [v2:172.21.15.159:6802/87273908,v1:172.21.15.159:6803/87273908]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2023-03-27T10:17:50.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:49 smithi161 ceph-mon[94333]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:50.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:50.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:50.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:17:50.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:50.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:50.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:50.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:50.114 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 27 10:17:49 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-0[106031]: 2023-03-27T10:17:49.941+0000 7f116d2c2700 -1 osd.0 0 waiting for initial osdmap 2023-03-27T10:17:50.115 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 27 10:17:49 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-0[106031]: 2023-03-27T10:17:49.946+0000 7f11688b6700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-03-27T10:17:50.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:49 smithi159 ceph-mon[93039]: from='osd.0 [v2:172.21.15.159:6802/87273908,v1:172.21.15.159:6803/87273908]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-03-27T10:17:50.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:49 smithi159 ceph-mon[93039]: osdmap e6: 1 total, 0 up, 1 in 2023-03-27T10:17:50.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-03-27T10:17:50.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:49 smithi159 ceph-mon[93039]: from='osd.0 [v2:172.21.15.159:6802/87273908,v1:172.21.15.159:6803/87273908]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2023-03-27T10:17:50.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:49 smithi159 ceph-mon[93039]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:50.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:50.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:50.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:17:50.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:50.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:50.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:50.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:50.142 DEBUG:teuthology.orchestra.run.smithi161:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch daemon add osd smithi161:vg_nvme/lv_4 2023-03-27T10:17:50.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:49 smithi171 ceph-mon[94808]: from='osd.0 [v2:172.21.15.159:6802/87273908,v1:172.21.15.159:6803/87273908]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-03-27T10:17:50.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:49 smithi171 ceph-mon[94808]: osdmap e6: 1 total, 0 up, 1 in 2023-03-27T10:17:50.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-03-27T10:17:50.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:49 smithi171 ceph-mon[94808]: from='osd.0 [v2:172.21.15.159:6802/87273908,v1:172.21.15.159:6803/87273908]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2023-03-27T10:17:50.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:49 smithi171 ceph-mon[94808]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-03-27T10:17:50.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:50.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:50.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:17:50.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:50.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:50.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:17:50.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:17:50.783 INFO:teuthology.orchestra.run.smithi161.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.b/config 2023-03-27T10:17:51.197 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:50 smithi161 ceph-mon[94333]: Detected new or changed devices on smithi159 2023-03-27T10:17:51.198 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:50 smithi161 ceph-mon[94333]: Adjusting osd_memory_target on smithi159 to 17168M 2023-03-27T10:17:51.198 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:50 smithi161 ceph-mon[94333]: from='osd.0 [v2:172.21.15.159:6802/87273908,v1:172.21.15.159:6803/87273908]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2023-03-27T10:17:51.198 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:50 smithi161 ceph-mon[94333]: osdmap e7: 1 total, 0 up, 1 in 2023-03-27T10:17:51.198 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:50 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-03-27T10:17:51.198 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:50 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-03-27T10:17:51.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:50 smithi171 ceph-mon[94808]: Detected new or changed devices on smithi159 2023-03-27T10:17:51.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:50 smithi171 ceph-mon[94808]: Adjusting osd_memory_target on smithi159 to 17168M 2023-03-27T10:17:51.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:50 smithi171 ceph-mon[94808]: from='osd.0 [v2:172.21.15.159:6802/87273908,v1:172.21.15.159:6803/87273908]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2023-03-27T10:17:51.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:50 smithi171 ceph-mon[94808]: osdmap e7: 1 total, 0 up, 1 in 2023-03-27T10:17:51.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:50 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-03-27T10:17:51.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:50 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-03-27T10:17:51.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:50 smithi159 ceph-mon[93039]: Detected new or changed devices on smithi159 2023-03-27T10:17:51.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:50 smithi159 ceph-mon[93039]: Adjusting osd_memory_target on smithi159 to 17168M 2023-03-27T10:17:51.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:50 smithi159 ceph-mon[93039]: from='osd.0 [v2:172.21.15.159:6802/87273908,v1:172.21.15.159:6803/87273908]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2023-03-27T10:17:51.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:50 smithi159 ceph-mon[93039]: osdmap e7: 1 total, 0 up, 1 in 2023-03-27T10:17:51.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:50 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-03-27T10:17:51.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:50 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-03-27T10:17:51.962 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.962+0000 7fe413fa9700 1 -- 172.21.15.161:0/1130157010 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe40c0ffe50 msgr2=0x7fe40c100260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:51.963 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.962+0000 7fe413fa9700 1 --2- 172.21.15.161:0/1130157010 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe40c0ffe50 0x7fe40c100260 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fe40400b0a0 tx=0x7fe40400b3b0 comp rx=0 tx=0).stop 2023-03-27T10:17:51.963 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.962+0000 7fe413fa9700 1 -- 172.21.15.161:0/1130157010 shutdown_connections 2023-03-27T10:17:51.963 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.962+0000 7fe413fa9700 1 --2- 172.21.15.161:0/1130157010 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe40c101960 0x7fe40c108290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:51.963 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.962+0000 7fe413fa9700 1 --2- 172.21.15.161:0/1130157010 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe40c0ffe50 0x7fe40c100260 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:51.963 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.962+0000 7fe413fa9700 1 --2- 172.21.15.161:0/1130157010 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe40c100fb0 0x7fe40c101420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:51.964 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.962+0000 7fe413fa9700 1 -- 172.21.15.161:0/1130157010 >> 172.21.15.161:0/1130157010 conn(0x7fe40c0fb480 msgr2=0x7fe40c0fd8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:17:51.964 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.962+0000 7fe413fa9700 1 -- 172.21.15.161:0/1130157010 shutdown_connections 2023-03-27T10:17:51.964 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.963+0000 7fe413fa9700 1 -- 172.21.15.161:0/1130157010 wait complete. 2023-03-27T10:17:51.965 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.963+0000 7fe413fa9700 1 Processor -- start 2023-03-27T10:17:51.965 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.963+0000 7fe413fa9700 1 -- start start 2023-03-27T10:17:51.966 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.963+0000 7fe413fa9700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe40c0ffe50 0x7fe40c195bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:51.966 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.963+0000 7fe413fa9700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe40c100fb0 0x7fe40c1960f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:51.966 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.963+0000 7fe413fa9700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe40c101960 0x7fe40c19d170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:51.966 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.963+0000 7fe413fa9700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe40c19d740 con 0x7fe40c100fb0 2023-03-27T10:17:51.966 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.963+0000 7fe413fa9700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe40c19d880 con 0x7fe40c101960 2023-03-27T10:17:51.966 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.963+0000 7fe413fa9700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe40c19da10 con 0x7fe40c0ffe50 2023-03-27T10:17:51.966 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.963+0000 7fe412546700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe40c101960 0x7fe40c19d170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:51.967 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.963+0000 7fe412546700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe40c101960 0x7fe40c19d170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.161:3300/0 says I am v2:172.21.15.161:44490/0 (socket says 172.21.15.161:44490) 2023-03-27T10:17:51.967 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.963+0000 7fe412546700 1 -- 172.21.15.161:0/4115480762 learned_addr learned my addr 172.21.15.161:0/4115480762 (peer_addr_for_me v2:172.21.15.161:0/0) 2023-03-27T10:17:51.967 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.963+0000 7fe411544700 1 --2- 172.21.15.161:0/4115480762 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe40c100fb0 0x7fe40c1960f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:51.967 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.964+0000 7fe411d45700 1 --2- 172.21.15.161:0/4115480762 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe40c0ffe50 0x7fe40c195bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:51.967 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.964+0000 7fe412546700 1 -- 172.21.15.161:0/4115480762 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe40c0ffe50 msgr2=0x7fe40c195bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:51.967 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.964+0000 7fe412546700 1 --2- 172.21.15.161:0/4115480762 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe40c0ffe50 0x7fe40c195bb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:51.967 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.964+0000 7fe412546700 1 -- 172.21.15.161:0/4115480762 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe40c100fb0 msgr2=0x7fe40c1960f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:17:51.967 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.964+0000 7fe412546700 1 --2- 172.21.15.161:0/4115480762 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe40c100fb0 0x7fe40c1960f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:17:51.968 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.964+0000 7fe412546700 1 -- 172.21.15.161:0/4115480762 --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe404009d00 con 0x7fe40c101960 2023-03-27T10:17:51.968 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.964+0000 7fe411d45700 1 --2- 172.21.15.161:0/4115480762 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe40c0ffe50 0x7fe40c195bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:17:51.968 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.964+0000 7fe412546700 1 --2- 172.21.15.161:0/4115480762 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe40c101960 0x7fe40c19d170 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fe40800d980 tx=0x7fe40800dc90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:17:51.968 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.964+0000 7fe402ffd700 1 -- 172.21.15.161:0/4115480762 <== mon.2 v2:172.21.15.161:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe408004220 con 0x7fe40c101960 2023-03-27T10:17:51.968 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.964+0000 7fe402ffd700 1 -- 172.21.15.161:0/4115480762 <== mon.2 v2:172.21.15.161:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe4080075e0 con 0x7fe40c101960 2023-03-27T10:17:51.968 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.964+0000 7fe413fa9700 1 -- 172.21.15.161:0/4115480762 --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe40c10b7a0 con 0x7fe40c101960 2023-03-27T10:17:51.968 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.964+0000 7fe402ffd700 1 -- 172.21.15.161:0/4115480762 <== mon.2 v2:172.21.15.161:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4080055f0 con 0x7fe40c101960 2023-03-27T10:17:51.968 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.964+0000 7fe413fa9700 1 -- 172.21.15.161:0/4115480762 --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe40c10b610 con 0x7fe40c101960 2023-03-27T10:17:51.969 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.965+0000 7fe413fa9700 1 -- 172.21.15.161:0/4115480762 --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe40c106d00 con 0x7fe40c101960 2023-03-27T10:17:51.969 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.966+0000 7fe402ffd700 1 -- 172.21.15.161:0/4115480762 <== mon.2 v2:172.21.15.161:3300/0 4 ==== mgrmap(e 15) v1 ==== 129818+0+0 (secure 0 0 0) 0x7fe408021070 con 0x7fe40c101960 2023-03-27T10:17:51.969 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.966+0000 7fe402ffd700 1 --2- 172.21.15.161:0/4115480762 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe3f809b350 0x7fe3f809d800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:17:51.970 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.966+0000 7fe402ffd700 1 -- 172.21.15.161:0/4115480762 <== mon.2 v2:172.21.15.161:3300/0 5 ==== osd_map(9..9 src has 1..9) v4 ==== 1767+0+0 (secure 0 0 0) 0x7fe4080c3240 con 0x7fe40c101960 2023-03-27T10:17:51.970 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.967+0000 7fe411d45700 1 --2- 172.21.15.161:0/4115480762 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe3f809b350 0x7fe3f809d800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:17:51.970 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.967+0000 7fe411d45700 1 --2- 172.21.15.161:0/4115480762 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe3f809b350 0x7fe3f809d800 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7fe404009850 tx=0x7fe404009780 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:17:51.971 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:51.971+0000 7fe402ffd700 1 -- 172.21.15.161:0/4115480762 <== mon.2 v2:172.21.15.161:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fe408090590 con 0x7fe40c101960 2023-03-27T10:17:52.158 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:17:52.158+0000 7fe413fa9700 1 -- 172.21.15.161:0/4115480762 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi161:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) v1 -- 0x7fe40c061870 con 0x7fe3f809b350 2023-03-27T10:17:52.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:51 smithi171 ceph-mon[94808]: purged_snaps scrub starts 2023-03-27T10:17:52.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:51 smithi171 ceph-mon[94808]: purged_snaps scrub ok 2023-03-27T10:17:52.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:51 smithi171 ceph-mon[94808]: osd.0 [v2:172.21.15.159:6802/87273908,v1:172.21.15.159:6803/87273908] boot 2023-03-27T10:17:52.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:51 smithi171 ceph-mon[94808]: osdmap e8: 1 total, 1 up, 1 in 2023-03-27T10:17:52.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:51 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-03-27T10:17:52.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:51 smithi171 ceph-mon[94808]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:17:52.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:51 smithi161 ceph-mon[94333]: purged_snaps scrub starts 2023-03-27T10:17:52.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:51 smithi161 ceph-mon[94333]: purged_snaps scrub ok 2023-03-27T10:17:52.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:51 smithi161 ceph-mon[94333]: osd.0 [v2:172.21.15.159:6802/87273908,v1:172.21.15.159:6803/87273908] boot 2023-03-27T10:17:52.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:51 smithi161 ceph-mon[94333]: osdmap e8: 1 total, 1 up, 1 in 2023-03-27T10:17:52.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:51 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-03-27T10:17:52.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:51 smithi161 ceph-mon[94333]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:17:52.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:51 smithi159 ceph-mon[93039]: purged_snaps scrub starts 2023-03-27T10:17:52.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:51 smithi159 ceph-mon[93039]: purged_snaps scrub ok 2023-03-27T10:17:52.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:51 smithi159 ceph-mon[93039]: osd.0 [v2:172.21.15.159:6802/87273908,v1:172.21.15.159:6803/87273908] boot 2023-03-27T10:17:52.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:51 smithi159 ceph-mon[93039]: osdmap e8: 1 total, 1 up, 1 in 2023-03-27T10:17:52.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:51 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-03-27T10:17:52.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:51 smithi159 ceph-mon[93039]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:17:53.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:52 smithi171 ceph-mon[94808]: osdmap e9: 1 total, 1 up, 1 in 2023-03-27T10:17:53.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:52 smithi171 ceph-mon[94808]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi161:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:17:53.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:52 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:17:53.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:52 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:17:53.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:52 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:53.228 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:52 smithi161 ceph-mon[94333]: osdmap e9: 1 total, 1 up, 1 in 2023-03-27T10:17:53.229 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:52 smithi161 ceph-mon[94333]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi161:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:17:53.229 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:52 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:17:53.229 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:52 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:17:53.229 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:52 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:53.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:52 smithi159 ceph-mon[93039]: osdmap e9: 1 total, 1 up, 1 in 2023-03-27T10:17:53.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:52 smithi159 ceph-mon[93039]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi161:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:17:53.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:52 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:17:53.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:52 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:17:53.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:52 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:17:54.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:53 smithi171 ceph-mon[94808]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:17:54.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:53 smithi161 ceph-mon[94333]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:17:54.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:53 smithi159 ceph-mon[93039]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:17:55.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:54 smithi171 ceph-mon[94808]: from='client.? 172.21.15.161:0/178396482' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97f66da9-22b4-4eb5-9c77-9d37cf0fca82"}]: dispatch 2023-03-27T10:17:55.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:54 smithi171 ceph-mon[94808]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97f66da9-22b4-4eb5-9c77-9d37cf0fca82"}]: dispatch 2023-03-27T10:17:55.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:54 smithi171 ceph-mon[94808]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "97f66da9-22b4-4eb5-9c77-9d37cf0fca82"}]': finished 2023-03-27T10:17:55.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:54 smithi171 ceph-mon[94808]: osdmap e10: 2 total, 1 up, 2 in 2023-03-27T10:17:55.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:54 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:17:55.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:54 smithi161 ceph-mon[94333]: from='client.? 172.21.15.161:0/178396482' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97f66da9-22b4-4eb5-9c77-9d37cf0fca82"}]: dispatch 2023-03-27T10:17:55.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:54 smithi161 ceph-mon[94333]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97f66da9-22b4-4eb5-9c77-9d37cf0fca82"}]: dispatch 2023-03-27T10:17:55.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:54 smithi161 ceph-mon[94333]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "97f66da9-22b4-4eb5-9c77-9d37cf0fca82"}]': finished 2023-03-27T10:17:55.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:54 smithi161 ceph-mon[94333]: osdmap e10: 2 total, 1 up, 2 in 2023-03-27T10:17:55.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:54 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:17:55.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:54 smithi159 ceph-mon[93039]: from='client.? 172.21.15.161:0/178396482' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97f66da9-22b4-4eb5-9c77-9d37cf0fca82"}]: dispatch 2023-03-27T10:17:55.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:54 smithi159 ceph-mon[93039]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97f66da9-22b4-4eb5-9c77-9d37cf0fca82"}]: dispatch 2023-03-27T10:17:55.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:54 smithi159 ceph-mon[93039]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "97f66da9-22b4-4eb5-9c77-9d37cf0fca82"}]': finished 2023-03-27T10:17:55.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:54 smithi159 ceph-mon[93039]: osdmap e10: 2 total, 1 up, 2 in 2023-03-27T10:17:55.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:54 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:17:56.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:55 smithi171 ceph-mon[94808]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:17:56.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:55 smithi171 ceph-mon[94808]: from='client.? 172.21.15.161:0/1308640824' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:17:56.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:55 smithi161 ceph-mon[94333]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:17:56.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:55 smithi161 ceph-mon[94333]: from='client.? 172.21.15.161:0/1308640824' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:17:56.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:55 smithi159 ceph-mon[93039]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:17:56.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:55 smithi159 ceph-mon[93039]: from='client.? 172.21.15.161:0/1308640824' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:17:58.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:57 smithi171 ceph-mon[94808]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:17:58.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:57 smithi161 ceph-mon[94333]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:17:58.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:57 smithi159 ceph-mon[93039]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:00.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:17:59 smithi161 ceph-mon[94333]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:00.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:17:59 smithi159 ceph-mon[93039]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:00.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:17:59 smithi171 ceph-mon[94808]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:01.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:01 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-03-27T10:18:01.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:01 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:01.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:01 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-03-27T10:18:01.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:01 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:01.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:01 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-03-27T10:18:01.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:01 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:02.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:02 smithi171 ceph-mon[94808]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:02.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:02 smithi171 ceph-mon[94808]: Deploying daemon osd.1 on smithi161 2023-03-27T10:18:02.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:02 smithi161 ceph-mon[94333]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:02.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:02 smithi161 ceph-mon[94333]: Deploying daemon osd.1 on smithi161 2023-03-27T10:18:02.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:02 smithi159 ceph-mon[93039]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:02.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:02 smithi159 ceph-mon[93039]: Deploying daemon osd.1 on smithi161 2023-03-27T10:18:04.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:04 smithi171 ceph-mon[94808]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:04.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:04 smithi161 ceph-mon[94333]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:04.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:04 smithi159 ceph-mon[93039]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:06.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:06 smithi171 ceph-mon[94808]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:06.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:06.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:06.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:06.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:06 smithi161 ceph-mon[94333]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:06.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:06.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:06.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:06.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:06 smithi159 ceph-mon[93039]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:06.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:06.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:06.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:08.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:08 smithi171 ceph-mon[94808]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:08.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:08.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:08.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:08 smithi161 ceph-mon[94333]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:08.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:08 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:08.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:08 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:08.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:08 smithi159 ceph-mon[93039]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:08.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:08.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:08.928 INFO:teuthology.orchestra.run.smithi161.stdout:Created osd(s) 1 on host 'smithi161' 2023-03-27T10:18:08.928 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:08.928+0000 7fe402ffd700 1 -- 172.21.15.161:0/4115480762 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7fe40c061870 con 0x7fe3f809b350 2023-03-27T10:18:08.930 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:08.931+0000 7fe413fa9700 1 -- 172.21.15.161:0/4115480762 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe3f809b350 msgr2=0x7fe3f809d800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:18:08.930 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:08.931+0000 7fe413fa9700 1 --2- 172.21.15.161:0/4115480762 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe3f809b350 0x7fe3f809d800 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7fe404009850 tx=0x7fe404009780 comp rx=0 tx=0).stop 2023-03-27T10:18:08.930 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:08.931+0000 7fe413fa9700 1 -- 172.21.15.161:0/4115480762 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe40c101960 msgr2=0x7fe40c19d170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:18:08.931 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:08.931+0000 7fe413fa9700 1 --2- 172.21.15.161:0/4115480762 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe40c101960 0x7fe40c19d170 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fe40800d980 tx=0x7fe40800dc90 comp rx=0 tx=0).stop 2023-03-27T10:18:08.931 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:08.931+0000 7fe413fa9700 1 -- 172.21.15.161:0/4115480762 shutdown_connections 2023-03-27T10:18:08.931 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:08.931+0000 7fe413fa9700 1 --2- 172.21.15.161:0/4115480762 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fe3f809b350 0x7fe3f809d800 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:08.931 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:08.931+0000 7fe413fa9700 1 --2- 172.21.15.161:0/4115480762 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fe40c0ffe50 0x7fe40c195bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:08.931 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:08.931+0000 7fe413fa9700 1 --2- 172.21.15.161:0/4115480762 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fe40c101960 0x7fe40c19d170 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:08.931 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:08.931+0000 7fe413fa9700 1 --2- 172.21.15.161:0/4115480762 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fe40c100fb0 0x7fe40c1960f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:08.931 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:08.931+0000 7fe413fa9700 1 -- 172.21.15.161:0/4115480762 >> 172.21.15.161:0/4115480762 conn(0x7fe40c0fb480 msgr2=0x7fe40c0fd680 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:18:08.932 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:08.931+0000 7fe413fa9700 1 -- 172.21.15.161:0/4115480762 shutdown_connections 2023-03-27T10:18:08.932 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:08.931+0000 7fe413fa9700 1 -- 172.21.15.161:0/4115480762 wait complete. 2023-03-27T10:18:09.359 DEBUG:teuthology.orchestra.run.smithi161:osd.1> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.1.service 2023-03-27T10:18:09.361 INFO:tasks.cephadm:Deploying osd.2 on smithi171 with /dev/vg_nvme/lv_4... 2023-03-27T10:18:09.362 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-03-27T10:18:09.390 INFO:journalctl@ceph.osd.1.smithi161.stdout:-- Logs begin at Mon 2023-03-27 10:02:59 UTC. -- 2023-03-27T10:18:09.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:09 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:09.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:09 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:09.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:09 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:09.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:09 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:09.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:09 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:09.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:09 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:09.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:09 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:09.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:09 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:09.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:09 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:09.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:09 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:09.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:09 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:09.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:09 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:09.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:09 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:09.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:09 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:09.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:09 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:09.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:09 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:09.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:09 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:09.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:09 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:10.004 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.c/config 2023-03-27T10:18:10.268 INFO:journalctl@ceph.osd.1.smithi161.stdout:Mar 27 10:18:09 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-1[99296]: 2023-03-27T10:18:09.947+0000 7f13dc367580 -1 osd.1 0 log_to_monitors true 2023-03-27T10:18:10.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:10 smithi171 ceph-mon[94808]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:10.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:10 smithi171 ceph-mon[94808]: from='osd.1 [v2:172.21.15.161:6800/2094166586,v1:172.21.15.161:6801/2094166586]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-03-27T10:18:10.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:10 smithi171 ceph-mon[94808]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-03-27T10:18:10.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:10 smithi161 ceph-mon[94333]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:10.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:10 smithi161 ceph-mon[94333]: from='osd.1 [v2:172.21.15.161:6800/2094166586,v1:172.21.15.161:6801/2094166586]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-03-27T10:18:10.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:10 smithi161 ceph-mon[94333]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-03-27T10:18:10.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:10 smithi159 ceph-mon[93039]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:10.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:10 smithi159 ceph-mon[93039]: from='osd.1 [v2:172.21.15.161:6800/2094166586,v1:172.21.15.161:6801/2094166586]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-03-27T10:18:10.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:10 smithi159 ceph-mon[93039]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-03-27T10:18:11.590 INFO:teuthology.orchestra.run.smithi171.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-03-27T10:18:11.590 INFO:teuthology.orchestra.run.smithi171.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-03-27T10:18:11.590 INFO:teuthology.orchestra.run.smithi171.stderr: stderr: 10+0 records in 2023-03-27T10:18:11.591 INFO:teuthology.orchestra.run.smithi171.stderr:10+0 records out 2023-03-27T10:18:11.591 INFO:teuthology.orchestra.run.smithi171.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110885 s, 946 MB/s 2023-03-27T10:18:11.591 INFO:teuthology.orchestra.run.smithi171.stderr:--> Zapping successful for: 2023-03-27T10:18:12.055 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch daemon add osd smithi171:vg_nvme/lv_4 2023-03-27T10:18:12.084 INFO:journalctl@ceph.osd.1.smithi161.stdout:Mar 27 10:18:11 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-1[99296]: 2023-03-27T10:18:11.943+0000 7f13d19e2700 -1 osd.1 0 waiting for initial osdmap 2023-03-27T10:18:12.084 INFO:journalctl@ceph.osd.1.smithi161.stdout:Mar 27 10:18:11 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-1[99296]: 2023-03-27T10:18:11.947+0000 7f13ccfd6700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-03-27T10:18:12.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:11 smithi161 ceph-mon[94333]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-03-27T10:18:12.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:11 smithi161 ceph-mon[94333]: osdmap e11: 2 total, 1 up, 2 in 2023-03-27T10:18:12.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:18:12.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:11 smithi161 ceph-mon[94333]: from='osd.1 [v2:172.21.15.161:6800/2094166586,v1:172.21.15.161:6801/2094166586]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:18:12.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:11 smithi161 ceph-mon[94333]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:18:12.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:11 smithi161 ceph-mon[94333]: pgmap v53: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:12.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:12.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:12.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:18:12.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:12.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:12.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:12.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:12.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:11 smithi171 ceph-mon[94808]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-03-27T10:18:12.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:11 smithi171 ceph-mon[94808]: osdmap e11: 2 total, 1 up, 2 in 2023-03-27T10:18:12.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:11 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:18:12.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:11 smithi171 ceph-mon[94808]: from='osd.1 [v2:172.21.15.161:6800/2094166586,v1:172.21.15.161:6801/2094166586]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:18:12.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:11 smithi171 ceph-mon[94808]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:18:12.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:11 smithi171 ceph-mon[94808]: pgmap v53: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:12.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:11 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:12.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:11 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:12.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:11 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:18:12.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:11 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:12.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:11 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:12.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:11 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:12.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:11 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:12.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:11 smithi159 ceph-mon[93039]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-03-27T10:18:12.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:11 smithi159 ceph-mon[93039]: osdmap e11: 2 total, 1 up, 2 in 2023-03-27T10:18:12.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:11 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:18:12.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:11 smithi159 ceph-mon[93039]: from='osd.1 [v2:172.21.15.161:6800/2094166586,v1:172.21.15.161:6801/2094166586]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:18:12.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:11 smithi159 ceph-mon[93039]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:18:12.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:11 smithi159 ceph-mon[93039]: pgmap v53: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:12.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:11 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:12.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:11 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:12.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:11 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:18:12.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:11 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:12.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:11 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:12.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:11 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:12.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:11 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:12.690 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.c/config 2023-03-27T10:18:13.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:12 smithi171 ceph-mon[94808]: Detected new or changed devices on smithi161 2023-03-27T10:18:13.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:12 smithi171 ceph-mon[94808]: Adjusting osd_memory_target on smithi161 to 17168M 2023-03-27T10:18:13.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:12 smithi171 ceph-mon[94808]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]': finished 2023-03-27T10:18:13.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:12 smithi171 ceph-mon[94808]: osdmap e12: 2 total, 1 up, 2 in 2023-03-27T10:18:13.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:12 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:18:13.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:12 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:18:13.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:12 smithi161 ceph-mon[94333]: Detected new or changed devices on smithi161 2023-03-27T10:18:13.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:12 smithi161 ceph-mon[94333]: Adjusting osd_memory_target on smithi161 to 17168M 2023-03-27T10:18:13.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:12 smithi161 ceph-mon[94333]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]': finished 2023-03-27T10:18:13.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:12 smithi161 ceph-mon[94333]: osdmap e12: 2 total, 1 up, 2 in 2023-03-27T10:18:13.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:12 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:18:13.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:12 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:18:13.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:12 smithi159 ceph-mon[93039]: Detected new or changed devices on smithi161 2023-03-27T10:18:13.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:12 smithi159 ceph-mon[93039]: Adjusting osd_memory_target on smithi161 to 17168M 2023-03-27T10:18:13.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:12 smithi159 ceph-mon[93039]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]': finished 2023-03-27T10:18:13.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:12 smithi159 ceph-mon[93039]: osdmap e12: 2 total, 1 up, 2 in 2023-03-27T10:18:13.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:12 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:18:13.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:12 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:18:13.926 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.925+0000 7f1ebd2a8700 1 -- 172.21.15.171:0/1884599793 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1eb81008d0 msgr2=0x7f1eb8107190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:18:13.926 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.925+0000 7f1ebd2a8700 1 --2- 172.21.15.171:0/1884599793 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1eb81008d0 0x7f1eb8107190 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f1eb000b0a0 tx=0x7f1eb000b3b0 comp rx=0 tx=0).stop 2023-03-27T10:18:13.926 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.925+0000 7f1ebd2a8700 1 -- 172.21.15.171:0/1884599793 shutdown_connections 2023-03-27T10:18:13.926 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.925+0000 7f1ebd2a8700 1 --2- 172.21.15.171:0/1884599793 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1eb81008d0 0x7f1eb8107190 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:13.926 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.925+0000 7f1ebd2a8700 1 --2- 172.21.15.171:0/1884599793 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1eb80fed40 0x7f1eb80ff150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:13.927 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.925+0000 7f1ebd2a8700 1 --2- 172.21.15.171:0/1884599793 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1eb80fff40 0x7f1eb8100390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:13.927 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.925+0000 7f1ebd2a8700 1 -- 172.21.15.171:0/1884599793 >> 172.21.15.171:0/1884599793 conn(0x7f1eb80fa370 msgr2=0x7f1eb80fc7c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:18:13.927 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.925+0000 7f1ebd2a8700 1 -- 172.21.15.171:0/1884599793 shutdown_connections 2023-03-27T10:18:13.927 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.925+0000 7f1ebd2a8700 1 -- 172.21.15.171:0/1884599793 wait complete. 2023-03-27T10:18:13.927 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.926+0000 7f1ebd2a8700 1 Processor -- start 2023-03-27T10:18:13.929 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.926+0000 7f1ebd2a8700 1 -- start start 2023-03-27T10:18:13.929 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1ebd2a8700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1eb80fed40 0x7f1eb8069790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:18:13.929 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1ebd2a8700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1eb80fff40 0x7f1eb819a8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:18:13.929 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1ebd2a8700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1eb81008d0 0x7f1eb819cf60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:18:13.929 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1ebd2a8700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1eb819d700 con 0x7f1eb81008d0 2023-03-27T10:18:13.930 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1ebd2a8700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1eb819d870 con 0x7f1eb80fff40 2023-03-27T10:18:13.930 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1ebd2a8700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1eb819da30 con 0x7f1eb80fed40 2023-03-27T10:18:13.930 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1eb6ffd700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1eb80fed40 0x7f1eb8069790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:18:13.930 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1eb6ffd700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1eb80fed40 0x7f1eb8069790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.171:3300/0 says I am v2:172.21.15.171:55130/0 (socket says 172.21.15.171:55130) 2023-03-27T10:18:13.930 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1eb6ffd700 1 -- 172.21.15.171:0/2668043786 learned_addr learned my addr 172.21.15.171:0/2668043786 (peer_addr_for_me v2:172.21.15.171:0/0) 2023-03-27T10:18:13.930 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1eb67fc700 1 --2- 172.21.15.171:0/2668043786 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1eb80fff40 0x7f1eb819a8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:18:13.930 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1eb6ffd700 1 -- 172.21.15.171:0/2668043786 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1eb80fff40 msgr2=0x7f1eb819a8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:18:13.931 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1eb6ffd700 1 --2- 172.21.15.171:0/2668043786 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1eb80fff40 0x7f1eb819a8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:13.931 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1eb6ffd700 1 -- 172.21.15.171:0/2668043786 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1eb81008d0 msgr2=0x7f1eb819cf60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:18:13.931 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1eb77fe700 1 --2- 172.21.15.171:0/2668043786 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1eb81008d0 0x7f1eb819cf60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:18:13.931 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1eb6ffd700 1 --2- 172.21.15.171:0/2668043786 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1eb81008d0 0x7f1eb819cf60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:13.931 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1eb6ffd700 1 -- 172.21.15.171:0/2668043786 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1eb0009d00 con 0x7f1eb80fed40 2023-03-27T10:18:13.931 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1eb67fc700 1 --2- 172.21.15.171:0/2668043786 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1eb80fff40 0x7f1eb819a8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-03-27T10:18:13.931 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1eb77fe700 1 --2- 172.21.15.171:0/2668043786 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1eb81008d0 0x7f1eb819cf60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-03-27T10:18:13.931 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1eb6ffd700 1 --2- 172.21.15.171:0/2668043786 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1eb80fed40 0x7f1eb8069790 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f1ea000e970 tx=0x7f1ea000ed30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:18:13.932 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1eaffff700 1 -- 172.21.15.171:0/2668043786 <== mon.1 v2:172.21.15.171:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ea0009790 con 0x7f1eb80fed40 2023-03-27T10:18:13.932 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1eaffff700 1 -- 172.21.15.171:0/2668043786 <== mon.1 v2:172.21.15.171:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1ea00044e0 con 0x7f1eb80fed40 2023-03-27T10:18:13.932 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1ebd2a8700 1 -- 172.21.15.171:0/2668043786 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1eb810a6a0 con 0x7f1eb80fed40 2023-03-27T10:18:13.932 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1eaffff700 1 -- 172.21.15.171:0/2668043786 <== mon.1 v2:172.21.15.171:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ea0003e30 con 0x7f1eb80fed40 2023-03-27T10:18:13.932 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.927+0000 7f1ebd2a8700 1 -- 172.21.15.171:0/2668043786 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1eb810a510 con 0x7f1eb80fed40 2023-03-27T10:18:13.932 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.928+0000 7f1ebd2a8700 1 -- 172.21.15.171:0/2668043786 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1eb81062d0 con 0x7f1eb80fed40 2023-03-27T10:18:13.932 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.929+0000 7f1eaffff700 1 -- 172.21.15.171:0/2668043786 <== mon.1 v2:172.21.15.171:3300/0 4 ==== mgrmap(e 15) v1 ==== 129818+0+0 (secure 0 0 0) 0x7f1ea001e020 con 0x7f1eb80fed40 2023-03-27T10:18:13.933 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.930+0000 7f1eaffff700 1 --2- 172.21.15.171:0/2668043786 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1ea409b350 0x7f1ea409d800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:18:13.933 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.930+0000 7f1eaffff700 1 -- 172.21.15.171:0/2668043786 <== mon.1 v2:172.21.15.171:3300/0 5 ==== osd_map(13..13 src has 1..13) v4 ==== 2309+0+0 (secure 0 0 0) 0x7f1ea0014070 con 0x7f1eb80fed40 2023-03-27T10:18:13.933 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.930+0000 7f1eb67fc700 1 --2- 172.21.15.171:0/2668043786 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1ea409b350 0x7f1ea409d800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:18:13.933 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.931+0000 7f1eb67fc700 1 --2- 172.21.15.171:0/2668043786 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1ea409b350 0x7f1ea409d800 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f1ea8005fd0 tx=0x7f1ea8005dc0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:18:13.934 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:13.934+0000 7f1eaffff700 1 -- 172.21.15.171:0/2668043786 <== mon.1 v2:172.21.15.171:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f1ea0090b40 con 0x7f1eb80fed40 2023-03-27T10:18:14.123 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:14.123+0000 7f1ebd2a8700 1 -- 172.21.15.171:0/2668043786 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) v1 -- 0x7f1eb8061920 con 0x7f1ea409b350 2023-03-27T10:18:14.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:13 smithi171 ceph-mon[94808]: purged_snaps scrub starts 2023-03-27T10:18:14.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:13 smithi171 ceph-mon[94808]: purged_snaps scrub ok 2023-03-27T10:18:14.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:13 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:18:14.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:13 smithi171 ceph-mon[94808]: osd.1 [v2:172.21.15.161:6800/2094166586,v1:172.21.15.161:6801/2094166586] boot 2023-03-27T10:18:14.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:13 smithi171 ceph-mon[94808]: osdmap e13: 2 total, 2 up, 2 in 2023-03-27T10:18:14.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:13 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:18:14.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:13 smithi171 ceph-mon[94808]: pgmap v56: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:14.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:13 smithi161 ceph-mon[94333]: purged_snaps scrub starts 2023-03-27T10:18:14.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:13 smithi161 ceph-mon[94333]: purged_snaps scrub ok 2023-03-27T10:18:14.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:13 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:18:14.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:13 smithi161 ceph-mon[94333]: osd.1 [v2:172.21.15.161:6800/2094166586,v1:172.21.15.161:6801/2094166586] boot 2023-03-27T10:18:14.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:13 smithi161 ceph-mon[94333]: osdmap e13: 2 total, 2 up, 2 in 2023-03-27T10:18:14.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:13 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:18:14.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:13 smithi161 ceph-mon[94333]: pgmap v56: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:14.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:13 smithi159 ceph-mon[93039]: purged_snaps scrub starts 2023-03-27T10:18:14.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:13 smithi159 ceph-mon[93039]: purged_snaps scrub ok 2023-03-27T10:18:14.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:13 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:18:14.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:13 smithi159 ceph-mon[93039]: osd.1 [v2:172.21.15.161:6800/2094166586,v1:172.21.15.161:6801/2094166586] boot 2023-03-27T10:18:14.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:13 smithi159 ceph-mon[93039]: osdmap e13: 2 total, 2 up, 2 in 2023-03-27T10:18:14.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:13 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-03-27T10:18:14.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:13 smithi159 ceph-mon[93039]: pgmap v56: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-03-27T10:18:15.207 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:14 smithi171 ceph-mon[94808]: osdmap e14: 2 total, 2 up, 2 in 2023-03-27T10:18:15.207 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:14 smithi171 ceph-mon[94808]: from='client.24160 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:18:15.208 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:14 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:18:15.208 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:14 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:18:15.208 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:14 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:15.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:14 smithi161 ceph-mon[94333]: osdmap e14: 2 total, 2 up, 2 in 2023-03-27T10:18:15.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:14 smithi161 ceph-mon[94333]: from='client.24160 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:18:15.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:14 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:18:15.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:14 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:18:15.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:14 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:15.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:14 smithi159 ceph-mon[93039]: osdmap e14: 2 total, 2 up, 2 in 2023-03-27T10:18:15.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:14 smithi159 ceph-mon[93039]: from='client.24160 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:18:15.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:14 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:18:15.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:14 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:18:15.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:14 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:16.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:16 smithi171 ceph-mon[94808]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:16.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:16 smithi161 ceph-mon[94333]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:16.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:16 smithi159 ceph-mon[93039]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:17.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:17 smithi171 ceph-mon[94808]: from='client.? 172.21.15.171:0/1330104049' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba9f5f82-3aef-42f7-a9e6-0e07aa7d8df1"}]: dispatch 2023-03-27T10:18:17.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:17 smithi171 ceph-mon[94808]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba9f5f82-3aef-42f7-a9e6-0e07aa7d8df1"}]: dispatch 2023-03-27T10:18:17.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:17 smithi171 ceph-mon[94808]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ba9f5f82-3aef-42f7-a9e6-0e07aa7d8df1"}]': finished 2023-03-27T10:18:17.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:17 smithi171 ceph-mon[94808]: osdmap e15: 3 total, 2 up, 3 in 2023-03-27T10:18:17.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:17 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-03-27T10:18:17.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:17 smithi171 ceph-mon[94808]: from='client.? 172.21.15.171:0/1241990002' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:18:17.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:17 smithi161 ceph-mon[94333]: from='client.? 172.21.15.171:0/1330104049' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba9f5f82-3aef-42f7-a9e6-0e07aa7d8df1"}]: dispatch 2023-03-27T10:18:17.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:17 smithi161 ceph-mon[94333]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba9f5f82-3aef-42f7-a9e6-0e07aa7d8df1"}]: dispatch 2023-03-27T10:18:17.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:17 smithi161 ceph-mon[94333]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ba9f5f82-3aef-42f7-a9e6-0e07aa7d8df1"}]': finished 2023-03-27T10:18:17.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:17 smithi161 ceph-mon[94333]: osdmap e15: 3 total, 2 up, 3 in 2023-03-27T10:18:17.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:17 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-03-27T10:18:17.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:17 smithi161 ceph-mon[94333]: from='client.? 172.21.15.171:0/1241990002' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:18:17.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:17 smithi159 ceph-mon[93039]: from='client.? 172.21.15.171:0/1330104049' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba9f5f82-3aef-42f7-a9e6-0e07aa7d8df1"}]: dispatch 2023-03-27T10:18:17.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:17 smithi159 ceph-mon[93039]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba9f5f82-3aef-42f7-a9e6-0e07aa7d8df1"}]: dispatch 2023-03-27T10:18:17.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:17 smithi159 ceph-mon[93039]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ba9f5f82-3aef-42f7-a9e6-0e07aa7d8df1"}]': finished 2023-03-27T10:18:17.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:17 smithi159 ceph-mon[93039]: osdmap e15: 3 total, 2 up, 3 in 2023-03-27T10:18:17.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:17 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-03-27T10:18:17.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:17 smithi159 ceph-mon[93039]: from='client.? 172.21.15.171:0/1241990002' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:18:18.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:18 smithi171 ceph-mon[94808]: pgmap v60: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:18.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:18 smithi161 ceph-mon[94333]: pgmap v60: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:18.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:18 smithi159 ceph-mon[93039]: pgmap v60: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:20.663 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:20 smithi171 ceph-mon[94808]: pgmap v61: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:20.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:20 smithi161 ceph-mon[94333]: pgmap v61: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:20.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:20 smithi159 ceph-mon[93039]: pgmap v61: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:22.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:22 smithi171 ceph-mon[94808]: pgmap v62: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:22.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:22 smithi161 ceph-mon[94333]: pgmap v62: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:22.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:22 smithi159 ceph-mon[93039]: pgmap v62: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:23.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-03-27T10:18:23.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:23.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-03-27T10:18:23.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:23.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-03-27T10:18:23.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:24.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:24 smithi171 ceph-mon[94808]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:24.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:24 smithi171 ceph-mon[94808]: Deploying daemon osd.2 on smithi171 2023-03-27T10:18:24.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:24 smithi161 ceph-mon[94333]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:24.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:24 smithi161 ceph-mon[94333]: Deploying daemon osd.2 on smithi171 2023-03-27T10:18:24.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:24 smithi159 ceph-mon[93039]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:24.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:24 smithi159 ceph-mon[93039]: Deploying daemon osd.2 on smithi171 2023-03-27T10:18:26.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:26 smithi171 ceph-mon[94808]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:26.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:26 smithi161 ceph-mon[94333]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:26.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:26 smithi159 ceph-mon[93039]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:28.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:28 smithi171 ceph-mon[94808]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:28.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:28.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:28.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:28.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:28 smithi161 ceph-mon[94333]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:28.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:28.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:28.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:28.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:28 smithi159 ceph-mon[93039]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:28.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:28.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:28.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:30.685 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:30 smithi171 ceph-mon[94808]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:30.685 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:30.685 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:30.686 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:30.686 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:30.686 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:30.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:30 smithi161 ceph-mon[94333]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:30.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:30.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:30.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:30.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:30.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:30.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:30 smithi159 ceph-mon[93039]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:30.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:30.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:30.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:30.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:30.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:30.920 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:30.920+0000 7f1eaffff700 1 -- 172.21.15.171:0/2668043786 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f1eb8061920 con 0x7f1ea409b350 2023-03-27T10:18:30.920 INFO:teuthology.orchestra.run.smithi171.stdout:Created osd(s) 2 on host 'smithi171' 2023-03-27T10:18:30.922 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:30.922+0000 7f1ebd2a8700 1 -- 172.21.15.171:0/2668043786 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1ea409b350 msgr2=0x7f1ea409d800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:18:30.922 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:30.922+0000 7f1ebd2a8700 1 --2- 172.21.15.171:0/2668043786 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1ea409b350 0x7f1ea409d800 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f1ea8005fd0 tx=0x7f1ea8005dc0 comp rx=0 tx=0).stop 2023-03-27T10:18:30.922 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:30.922+0000 7f1ebd2a8700 1 -- 172.21.15.171:0/2668043786 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1eb80fed40 msgr2=0x7f1eb8069790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:18:30.922 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:30.922+0000 7f1ebd2a8700 1 --2- 172.21.15.171:0/2668043786 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1eb80fed40 0x7f1eb8069790 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f1ea000e970 tx=0x7f1ea000ed30 comp rx=0 tx=0).stop 2023-03-27T10:18:30.922 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:30.922+0000 7f1ebd2a8700 1 -- 172.21.15.171:0/2668043786 shutdown_connections 2023-03-27T10:18:30.923 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:30.922+0000 7f1ebd2a8700 1 --2- 172.21.15.171:0/2668043786 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1ea409b350 0x7f1ea409d800 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:30.923 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:30.922+0000 7f1ebd2a8700 1 --2- 172.21.15.171:0/2668043786 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1eb80fed40 0x7f1eb8069790 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:30.923 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:30.922+0000 7f1ebd2a8700 1 --2- 172.21.15.171:0/2668043786 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1eb81008d0 0x7f1eb819cf60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:30.923 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:30.922+0000 7f1ebd2a8700 1 --2- 172.21.15.171:0/2668043786 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1eb80fff40 0x7f1eb819a8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:30.923 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:30.922+0000 7f1ebd2a8700 1 -- 172.21.15.171:0/2668043786 >> 172.21.15.171:0/2668043786 conn(0x7f1eb80fa370 msgr2=0x7f1eb8103410 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:18:30.923 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:30.922+0000 7f1ebd2a8700 1 -- 172.21.15.171:0/2668043786 shutdown_connections 2023-03-27T10:18:30.923 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:18:30.922+0000 7f1ebd2a8700 1 -- 172.21.15.171:0/2668043786 wait complete. 2023-03-27T10:18:31.488 DEBUG:teuthology.orchestra.run.smithi171:osd.2> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.2.service 2023-03-27T10:18:31.490 INFO:tasks.cephadm:Deploying osd.3 on smithi159 with /dev/vg_nvme/lv_3... 2023-03-27T10:18:31.490 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-03-27T10:18:31.521 INFO:journalctl@ceph.osd.2.smithi171.stdout:-- Logs begin at Mon 2023-03-27 10:03:15 UTC. -- 2023-03-27T10:18:32.144 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:18:32.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:31 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:32.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:31 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:32.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:31 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:32.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:31 smithi171 ceph-mon[94808]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:32.227 INFO:journalctl@ceph.osd.2.smithi171.stdout:Mar 27 10:18:31 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-2[100593]: 2023-03-27T10:18:31.936+0000 7fe4319c3580 -1 osd.2 0 log_to_monitors true 2023-03-27T10:18:32.333 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:31 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:32.334 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:31 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:32.334 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:31 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:32.334 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:31 smithi159 ceph-mon[93039]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:32.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:31 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:32.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:31 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:32.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:31 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:32.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:31 smithi161 ceph-mon[94333]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:33.175 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:32 smithi159 ceph-mon[93039]: from='osd.2 [v2:172.21.15.171:6800/2724827688,v1:172.21.15.171:6801/2724827688]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-03-27T10:18:33.175 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:32 smithi159 ceph-mon[93039]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-03-27T10:18:33.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:32 smithi171 ceph-mon[94808]: from='osd.2 [v2:172.21.15.171:6800/2724827688,v1:172.21.15.171:6801/2724827688]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-03-27T10:18:33.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:32 smithi171 ceph-mon[94808]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-03-27T10:18:33.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:32 smithi161 ceph-mon[94333]: from='osd.2 [v2:172.21.15.171:6800/2724827688,v1:172.21.15.171:6801/2724827688]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-03-27T10:18:33.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:32 smithi161 ceph-mon[94333]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-03-27T10:18:33.729 INFO:teuthology.orchestra.run.smithi159.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-03-27T10:18:33.729 INFO:teuthology.orchestra.run.smithi159.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-03-27T10:18:33.729 INFO:teuthology.orchestra.run.smithi159.stderr: stderr: 10+0 records in 2023-03-27T10:18:33.730 INFO:teuthology.orchestra.run.smithi159.stderr:10+0 records out 2023-03-27T10:18:33.730 INFO:teuthology.orchestra.run.smithi159.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110845 s, 946 MB/s 2023-03-27T10:18:33.730 INFO:teuthology.orchestra.run.smithi159.stderr:--> Zapping successful for: 2023-03-27T10:18:34.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:33 smithi159 ceph-mon[93039]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-03-27T10:18:34.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:33 smithi159 ceph-mon[93039]: osdmap e16: 3 total, 2 up, 3 in 2023-03-27T10:18:34.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-03-27T10:18:34.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:33 smithi159 ceph-mon[93039]: from='osd.2 [v2:172.21.15.171:6800/2724827688,v1:172.21.15.171:6801/2724827688]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:18:34.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:33 smithi159 ceph-mon[93039]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:18:34.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:33 smithi159 ceph-mon[93039]: pgmap v69: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:34.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:34.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:34.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:18:34.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:34.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:34.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:34.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:34.189 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch daemon add osd smithi159:vg_nvme/lv_3 2023-03-27T10:18:34.226 INFO:journalctl@ceph.osd.2.smithi171.stdout:Mar 27 10:18:33 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-2[100593]: 2023-03-27T10:18:33.936+0000 7fe42703e700 -1 osd.2 0 waiting for initial osdmap 2023-03-27T10:18:34.226 INFO:journalctl@ceph.osd.2.smithi171.stdout:Mar 27 10:18:33 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-2[100593]: 2023-03-27T10:18:33.941+0000 7fe42062e700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-03-27T10:18:34.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:33 smithi171 ceph-mon[94808]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-03-27T10:18:34.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:33 smithi171 ceph-mon[94808]: osdmap e16: 3 total, 2 up, 3 in 2023-03-27T10:18:34.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-03-27T10:18:34.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:33 smithi171 ceph-mon[94808]: from='osd.2 [v2:172.21.15.171:6800/2724827688,v1:172.21.15.171:6801/2724827688]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:18:34.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:33 smithi171 ceph-mon[94808]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:18:34.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:33 smithi171 ceph-mon[94808]: pgmap v69: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:34.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:34.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:34.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:18:34.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:34.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:34.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:34.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:34.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:33 smithi161 ceph-mon[94333]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-03-27T10:18:34.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:33 smithi161 ceph-mon[94333]: osdmap e16: 3 total, 2 up, 3 in 2023-03-27T10:18:34.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-03-27T10:18:34.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:33 smithi161 ceph-mon[94333]: from='osd.2 [v2:172.21.15.171:6800/2724827688,v1:172.21.15.171:6801/2724827688]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:18:34.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:33 smithi161 ceph-mon[94333]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:18:34.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:33 smithi161 ceph-mon[94333]: pgmap v69: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-03-27T10:18:34.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:34.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:34.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:18:34.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:34.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:34.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:34.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:34.827 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:18:35.190 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:34 smithi159 ceph-mon[93039]: Detected new or changed devices on smithi171 2023-03-27T10:18:35.190 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:34 smithi159 ceph-mon[93039]: Adjusting osd_memory_target on smithi171 to 17168M 2023-03-27T10:18:35.190 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:34 smithi159 ceph-mon[93039]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2023-03-27T10:18:35.190 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:34 smithi159 ceph-mon[93039]: osdmap e17: 3 total, 2 up, 3 in 2023-03-27T10:18:35.190 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:34 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-03-27T10:18:35.191 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:34 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-03-27T10:18:35.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:34 smithi171 ceph-mon[94808]: Detected new or changed devices on smithi171 2023-03-27T10:18:35.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:34 smithi171 ceph-mon[94808]: Adjusting osd_memory_target on smithi171 to 17168M 2023-03-27T10:18:35.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:34 smithi171 ceph-mon[94808]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2023-03-27T10:18:35.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:34 smithi171 ceph-mon[94808]: osdmap e17: 3 total, 2 up, 3 in 2023-03-27T10:18:35.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:34 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-03-27T10:18:35.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:34 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-03-27T10:18:35.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:34 smithi161 ceph-mon[94333]: Detected new or changed devices on smithi171 2023-03-27T10:18:35.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:34 smithi161 ceph-mon[94333]: Adjusting osd_memory_target on smithi171 to 17168M 2023-03-27T10:18:35.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:34 smithi161 ceph-mon[94333]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2023-03-27T10:18:35.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:34 smithi161 ceph-mon[94333]: osdmap e17: 3 total, 2 up, 3 in 2023-03-27T10:18:35.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:34 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-03-27T10:18:35.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:34 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-03-27T10:18:35.916 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.916+0000 7f7a81c23700 1 -- 172.21.15.159:0/558629547 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7a7c0feca0 msgr2=0x7f7a7c0ff0b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:18:35.917 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.916+0000 7f7a81c23700 1 --2- 172.21.15.159:0/558629547 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7a7c0feca0 0x7f7a7c0ff0b0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f7a70009990 tx=0x7f7a70009ca0 comp rx=0 tx=0).stop 2023-03-27T10:18:35.917 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.916+0000 7f7a81c23700 1 -- 172.21.15.159:0/558629547 shutdown_connections 2023-03-27T10:18:35.917 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.916+0000 7f7a81c23700 1 --2- 172.21.15.159:0/558629547 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7a7c100830 0x7f7a7c1070f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:35.917 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.916+0000 7f7a81c23700 1 --2- 172.21.15.159:0/558629547 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7a7c0feca0 0x7f7a7c0ff0b0 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:35.917 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.916+0000 7f7a81c23700 1 --2- 172.21.15.159:0/558629547 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7a7c0ffea0 0x7f7a7c1002f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:35.918 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.916+0000 7f7a81c23700 1 -- 172.21.15.159:0/558629547 >> 172.21.15.159:0/558629547 conn(0x7f7a7c0fa2b0 msgr2=0x7f7a7c0fc720 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:18:35.918 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.916+0000 7f7a81c23700 1 -- 172.21.15.159:0/558629547 shutdown_connections 2023-03-27T10:18:35.918 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.916+0000 7f7a81c23700 1 -- 172.21.15.159:0/558629547 wait complete. 2023-03-27T10:18:35.918 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.917+0000 7f7a81c23700 1 Processor -- start 2023-03-27T10:18:35.918 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.917+0000 7f7a81c23700 1 -- start start 2023-03-27T10:18:35.918 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.917+0000 7f7a81c23700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7a7c0feca0 0x7f7a7c197d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:18:35.918 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.917+0000 7f7a81c23700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7a7c0ffea0 0x7f7a7c198270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:18:35.919 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.917+0000 7f7a81c23700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7a7c100830 0x7f7a7c19f2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:18:35.919 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.917+0000 7f7a81c23700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a7c19f880 con 0x7f7a7c0feca0 2023-03-27T10:18:35.919 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.917+0000 7f7a81c23700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a7c19f9c0 con 0x7f7a7c0ffea0 2023-03-27T10:18:35.919 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.917+0000 7f7a81c23700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a7c19fb80 con 0x7f7a7c100830 2023-03-27T10:18:35.919 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.917+0000 7f7a7b7fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7a7c0feca0 0x7f7a7c197d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:18:35.919 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.917+0000 7f7a7b7fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7a7c0feca0 0x7f7a7c197d30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44824/0 (socket says 172.21.15.159:44824) 2023-03-27T10:18:35.919 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.917+0000 7f7a7b7fe700 1 -- 172.21.15.159:0/2202741136 learned_addr learned my addr 172.21.15.159:0/2202741136 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:18:35.920 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.917+0000 7f7a7affd700 1 --2- 172.21.15.159:0/2202741136 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7a7c0ffea0 0x7f7a7c198270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:18:35.920 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.918+0000 7f7a7bfff700 1 --2- 172.21.15.159:0/2202741136 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7a7c100830 0x7f7a7c19f2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:18:35.920 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.918+0000 7f7a7b7fe700 1 -- 172.21.15.159:0/2202741136 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7a7c100830 msgr2=0x7f7a7c19f2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:18:35.920 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.918+0000 7f7a7b7fe700 1 --2- 172.21.15.159:0/2202741136 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7a7c100830 0x7f7a7c19f2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:35.920 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.918+0000 7f7a7b7fe700 1 -- 172.21.15.159:0/2202741136 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7a7c0ffea0 msgr2=0x7f7a7c198270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:18:35.920 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.918+0000 7f7a7b7fe700 1 --2- 172.21.15.159:0/2202741136 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7a7c0ffea0 0x7f7a7c198270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:35.920 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.918+0000 7f7a7b7fe700 1 -- 172.21.15.159:0/2202741136 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7a700095f0 con 0x7f7a7c0feca0 2023-03-27T10:18:35.921 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.918+0000 7f7a7bfff700 1 --2- 172.21.15.159:0/2202741136 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7a7c100830 0x7f7a7c19f2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-03-27T10:18:35.921 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.918+0000 7f7a7b7fe700 1 --2- 172.21.15.159:0/2202741136 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7a7c0feca0 0x7f7a7c197d30 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f7a700095c0 tx=0x7f7a7000f690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:18:35.921 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.918+0000 7f7a78ff9700 1 -- 172.21.15.159:0/2202741136 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a7001d070 con 0x7f7a7c0feca0 2023-03-27T10:18:35.921 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.918+0000 7f7a81c23700 1 -- 172.21.15.159:0/2202741136 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7a7c10a5f0 con 0x7f7a7c0feca0 2023-03-27T10:18:35.921 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.918+0000 7f7a78ff9700 1 -- 172.21.15.159:0/2202741136 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7a70022470 con 0x7f7a7c0feca0 2023-03-27T10:18:35.921 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.918+0000 7f7a78ff9700 1 -- 172.21.15.159:0/2202741136 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a70017650 con 0x7f7a7c0feca0 2023-03-27T10:18:35.921 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.918+0000 7f7a81c23700 1 -- 172.21.15.159:0/2202741136 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7a7c10a460 con 0x7f7a7c0feca0 2023-03-27T10:18:35.922 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.919+0000 7f7a81c23700 1 -- 172.21.15.159:0/2202741136 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7a7c1a00e0 con 0x7f7a7c0feca0 2023-03-27T10:18:35.922 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.920+0000 7f7a78ff9700 1 -- 172.21.15.159:0/2202741136 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 15) v1 ==== 129818+0+0 (secure 0 0 0) 0x7f7a70029030 con 0x7f7a7c0feca0 2023-03-27T10:18:35.923 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.920+0000 7f7a78ff9700 1 --2- 172.21.15.159:0/2202741136 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7a6409b350 0x7f7a6409d800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:18:35.923 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.921+0000 7f7a7affd700 1 --2- 172.21.15.159:0/2202741136 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7a6409b350 0x7f7a6409d800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:18:35.923 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.921+0000 7f7a78ff9700 1 -- 172.21.15.159:0/2202741136 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(18..18 src has 1..18) v4 ==== 2851+0+0 (secure 0 0 0) 0x7f7a700c5170 con 0x7f7a7c0feca0 2023-03-27T10:18:35.923 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.921+0000 7f7a7affd700 1 --2- 172.21.15.159:0/2202741136 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7a6409b350 0x7f7a6409d800 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f7a6c005950 tx=0x7f7a6c00b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:18:35.923 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:35.924+0000 7f7a78ff9700 1 -- 172.21.15.159:0/2202741136 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f7a70092110 con 0x7f7a7c0feca0 2023-03-27T10:18:36.118 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:36.119+0000 7f7a81c23700 1 -- 172.21.15.159:0/2202741136 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) v1 -- 0x7f7a7c068870 con 0x7f7a6409b350 2023-03-27T10:18:36.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:35 smithi171 ceph-mon[94808]: purged_snaps scrub starts 2023-03-27T10:18:36.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:35 smithi171 ceph-mon[94808]: purged_snaps scrub ok 2023-03-27T10:18:36.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:35 smithi171 ceph-mon[94808]: osd.2 [v2:172.21.15.171:6800/2724827688,v1:172.21.15.171:6801/2724827688] boot 2023-03-27T10:18:36.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:35 smithi171 ceph-mon[94808]: osdmap e18: 3 total, 3 up, 3 in 2023-03-27T10:18:36.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:35 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-03-27T10:18:36.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:35 smithi171 ceph-mon[94808]: pgmap v72: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:36.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:35 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2023-03-27T10:18:36.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:35 smithi161 ceph-mon[94333]: purged_snaps scrub starts 2023-03-27T10:18:36.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:35 smithi161 ceph-mon[94333]: purged_snaps scrub ok 2023-03-27T10:18:36.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:35 smithi161 ceph-mon[94333]: osd.2 [v2:172.21.15.171:6800/2724827688,v1:172.21.15.171:6801/2724827688] boot 2023-03-27T10:18:36.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:35 smithi161 ceph-mon[94333]: osdmap e18: 3 total, 3 up, 3 in 2023-03-27T10:18:36.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:35 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-03-27T10:18:36.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:35 smithi161 ceph-mon[94333]: pgmap v72: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:36.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:35 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2023-03-27T10:18:36.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:35 smithi159 ceph-mon[93039]: purged_snaps scrub starts 2023-03-27T10:18:36.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:35 smithi159 ceph-mon[93039]: purged_snaps scrub ok 2023-03-27T10:18:36.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:35 smithi159 ceph-mon[93039]: osd.2 [v2:172.21.15.171:6800/2724827688,v1:172.21.15.171:6801/2724827688] boot 2023-03-27T10:18:36.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:35 smithi159 ceph-mon[93039]: osdmap e18: 3 total, 3 up, 3 in 2023-03-27T10:18:36.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:35 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-03-27T10:18:36.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:35 smithi159 ceph-mon[93039]: pgmap v72: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:36.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:35 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2023-03-27T10:18:37.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:36 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2023-03-27T10:18:37.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:36 smithi171 ceph-mon[94808]: osdmap e19: 3 total, 3 up, 3 in 2023-03-27T10:18:37.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:36 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-03-27T10:18:37.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:36 smithi171 ceph-mon[94808]: from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:18:37.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:36 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:18:37.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:36 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:18:37.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:36 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:37.288 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2023-03-27T10:18:37.289 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:36 smithi159 ceph-mon[93039]: osdmap e19: 3 total, 3 up, 3 in 2023-03-27T10:18:37.289 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-03-27T10:18:37.289 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:36 smithi159 ceph-mon[93039]: from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:18:37.289 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:18:37.289 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:18:37.289 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:37.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:36 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2023-03-27T10:18:37.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:36 smithi161 ceph-mon[94333]: osdmap e19: 3 total, 3 up, 3 in 2023-03-27T10:18:37.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:36 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-03-27T10:18:37.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:36 smithi161 ceph-mon[94333]: from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:18:37.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:36 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:18:37.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:36 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:18:37.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:36 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:37.567 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 27 10:18:37 smithi159 sudo[108968]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-03-27T10:18:37.568 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 27 10:18:37 smithi159 sudo[108968]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-03-27T10:18:37.568 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 27 10:18:37 smithi159 sudo[108968]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-03-27T10:18:37.568 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 27 10:18:37 smithi159 sudo[108968]: pam_unix(sudo:session): session closed for user root 2023-03-27T10:18:37.864 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 27 10:18:37 smithi159 sudo[108989]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-03-27T10:18:37.865 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 27 10:18:37 smithi159 sudo[108989]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-03-27T10:18:37.865 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 27 10:18:37 smithi159 sudo[108989]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-03-27T10:18:37.865 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 27 10:18:37 smithi159 sudo[108989]: pam_unix(sudo:session): session closed for user root 2023-03-27T10:18:37.966 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:37.966+0000 7f7a78ff9700 1 -- 172.21.15.159:0/2202741136 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f7a7000c040 con 0x7f7a7c0feca0 2023-03-27T10:18:38.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:37 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-03-27T10:18:38.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:37 smithi161 ceph-mon[94333]: osdmap e20: 3 total, 3 up, 3 in 2023-03-27T10:18:38.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:37 smithi161 ceph-mon[94333]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:38.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:37 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:38.088 INFO:journalctl@ceph.osd.1.smithi161.stdout:Mar 27 10:18:37 smithi161 sudo[101688]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-03-27T10:18:38.088 INFO:journalctl@ceph.osd.1.smithi161.stdout:Mar 27 10:18:37 smithi161 sudo[101688]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-03-27T10:18:38.088 INFO:journalctl@ceph.osd.1.smithi161.stdout:Mar 27 10:18:37 smithi161 sudo[101688]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-03-27T10:18:38.088 INFO:journalctl@ceph.osd.1.smithi161.stdout:Mar 27 10:18:37 smithi161 sudo[101688]: pam_unix(sudo:session): session closed for user root 2023-03-27T10:18:38.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:37 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-03-27T10:18:38.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:37 smithi171 ceph-mon[94808]: osdmap e20: 3 total, 3 up, 3 in 2023-03-27T10:18:38.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:37 smithi171 ceph-mon[94808]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:38.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:37 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:38.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:37 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-03-27T10:18:38.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:37 smithi159 ceph-mon[93039]: osdmap e20: 3 total, 3 up, 3 in 2023-03-27T10:18:38.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:37 smithi159 ceph-mon[93039]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:38.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:37 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:38.584 INFO:journalctl@ceph.osd.1.smithi161.stdout:Mar 27 10:18:38 smithi161 sudo[101691]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-03-27T10:18:38.584 INFO:journalctl@ceph.osd.1.smithi161.stdout:Mar 27 10:18:38 smithi161 sudo[101691]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-03-27T10:18:38.584 INFO:journalctl@ceph.osd.1.smithi161.stdout:Mar 27 10:18:38 smithi161 sudo[101691]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-03-27T10:18:38.585 INFO:journalctl@ceph.osd.1.smithi161.stdout:Mar 27 10:18:38 smithi161 sudo[101691]: pam_unix(sudo:session): session closed for user root 2023-03-27T10:18:38.608 INFO:journalctl@ceph.osd.2.smithi171.stdout:Mar 27 10:18:38 smithi171 sudo[102828]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-03-27T10:18:38.608 INFO:journalctl@ceph.osd.2.smithi171.stdout:Mar 27 10:18:38 smithi171 sudo[102828]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-03-27T10:18:38.608 INFO:journalctl@ceph.osd.2.smithi171.stdout:Mar 27 10:18:38 smithi171 sudo[102828]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-03-27T10:18:38.608 INFO:journalctl@ceph.osd.2.smithi171.stdout:Mar 27 10:18:38 smithi171 sudo[102828]: pam_unix(sudo:session): session closed for user root 2023-03-27T10:18:38.976 INFO:journalctl@ceph.osd.2.smithi171.stdout:Mar 27 10:18:38 smithi171 sudo[102831]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-03-27T10:18:38.976 INFO:journalctl@ceph.osd.2.smithi171.stdout:Mar 27 10:18:38 smithi171 sudo[102831]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-03-27T10:18:38.977 INFO:journalctl@ceph.osd.2.smithi171.stdout:Mar 27 10:18:38 smithi171 sudo[102831]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-03-27T10:18:38.977 INFO:journalctl@ceph.osd.2.smithi171.stdout:Mar 27 10:18:38 smithi171 sudo[102831]: pam_unix(sudo:session): session closed for user root 2023-03-27T10:18:39.146 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:38 smithi159 sudo[109087]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-03-27T10:18:39.147 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:38 smithi159 sudo[109087]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-03-27T10:18:39.147 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:38 smithi159 sudo[109087]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-03-27T10:18:39.147 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:38 smithi159 sudo[109087]: pam_unix(sudo:session): session closed for user root 2023-03-27T10:18:39.406 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 ceph-mon[94808]: osdmap e21: 3 total, 3 up, 3 in 2023-03-27T10:18:39.406 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 ceph-mon[94808]: mgrmap e16: x(active, since 2m), standbys: z, y 2023-03-27T10:18:39.407 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:39.407 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3655021017' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ef4b519-fdef-4d38-b387-bc14ea381bd3"}]: dispatch 2023-03-27T10:18:39.407 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3655021017' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4ef4b519-fdef-4d38-b387-bc14ea381bd3"}]': finished 2023-03-27T10:18:39.407 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 ceph-mon[94808]: osdmap e22: 4 total, 3 up, 4 in 2023-03-27T10:18:39.407 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-03-27T10:18:39.407 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:39.407 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 ceph-mon[94808]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-03-27T10:18:39.408 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 sudo[102834]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-03-27T10:18:39.408 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 sudo[102834]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-03-27T10:18:39.408 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 sudo[102834]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-03-27T10:18:39.408 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 sudo[102834]: pam_unix(sudo:session): session closed for user root 2023-03-27T10:18:39.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:39 smithi161 ceph-mon[94333]: osdmap e21: 3 total, 3 up, 3 in 2023-03-27T10:18:39.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:39 smithi161 ceph-mon[94333]: mgrmap e16: x(active, since 2m), standbys: z, y 2023-03-27T10:18:39.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:39 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:39.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:39 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3655021017' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ef4b519-fdef-4d38-b387-bc14ea381bd3"}]: dispatch 2023-03-27T10:18:39.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:39 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3655021017' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4ef4b519-fdef-4d38-b387-bc14ea381bd3"}]': finished 2023-03-27T10:18:39.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:39 smithi161 ceph-mon[94333]: osdmap e22: 4 total, 3 up, 4 in 2023-03-27T10:18:39.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:39 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-03-27T10:18:39.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:39 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:39.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:39 smithi161 ceph-mon[94333]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-03-27T10:18:39.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:39 smithi159 sudo[109108]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-03-27T10:18:39.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:39 smithi159 sudo[109108]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-03-27T10:18:39.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:39 smithi159 sudo[109108]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-03-27T10:18:39.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:39 smithi159 sudo[109108]: pam_unix(sudo:session): session closed for user root 2023-03-27T10:18:39.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:39 smithi159 ceph-mon[93039]: osdmap e21: 3 total, 3 up, 3 in 2023-03-27T10:18:39.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:39 smithi159 ceph-mon[93039]: mgrmap e16: x(active, since 2m), standbys: z, y 2023-03-27T10:18:39.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:39 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:39.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:39 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3655021017' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ef4b519-fdef-4d38-b387-bc14ea381bd3"}]: dispatch 2023-03-27T10:18:39.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:39 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3655021017' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4ef4b519-fdef-4d38-b387-bc14ea381bd3"}]': finished 2023-03-27T10:18:39.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:39 smithi159 ceph-mon[93039]: osdmap e22: 4 total, 3 up, 4 in 2023-03-27T10:18:39.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:39 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-03-27T10:18:39.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:39 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:39.617 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:39 smithi159 ceph-mon[93039]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-03-27T10:18:39.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 sudo[102837]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-03-27T10:18:39.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 sudo[102837]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-03-27T10:18:39.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 sudo[102837]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-03-27T10:18:39.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:39 smithi171 sudo[102837]: pam_unix(sudo:session): session closed for user root 2023-03-27T10:18:40.138 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:39 smithi161 sudo[101694]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-03-27T10:18:40.138 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:39 smithi161 sudo[101694]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-03-27T10:18:40.138 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:39 smithi161 sudo[101694]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-03-27T10:18:40.139 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:39 smithi161 sudo[101694]: pam_unix(sudo:session): session closed for user root 2023-03-27T10:18:40.139 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 sudo[101697]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-03-27T10:18:40.139 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 sudo[101697]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-03-27T10:18:40.139 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 sudo[101697]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-03-27T10:18:40.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:40 smithi171 ceph-mon[94808]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:40.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:40 smithi171 ceph-mon[94808]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-03-27T10:18:40.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:40 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:18:40.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:40 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:18:40.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:40 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:18:40.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:40 smithi171 ceph-mon[94808]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-03-27T10:18:40.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:40 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:18:40.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:40 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:18:40.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:40 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:18:40.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:40 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/4006111262' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:18:40.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:40 smithi171 ceph-mon[94808]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-03-27T10:18:40.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:40 smithi171 ceph-mon[94808]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-03-27T10:18:40.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:40 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:18:40.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:40 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:18:40.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:40 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:18:40.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:40 smithi171 ceph-mon[94808]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-03-27T10:18:40.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 sudo[101697]: pam_unix(sudo:session): session closed for user root 2023-03-27T10:18:40.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 ceph-mon[94333]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:40.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 ceph-mon[94333]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-03-27T10:18:40.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:18:40.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:18:40.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:18:40.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 ceph-mon[94333]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-03-27T10:18:40.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:18:40.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:18:40.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:18:40.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/4006111262' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:18:40.586 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 ceph-mon[94333]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-03-27T10:18:40.586 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 ceph-mon[94333]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-03-27T10:18:40.586 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:18:40.586 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:18:40.586 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:18:40.586 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:40 smithi161 ceph-mon[94333]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-03-27T10:18:40.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:40 smithi159 ceph-mon[93039]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:40.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:40 smithi159 ceph-mon[93039]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-03-27T10:18:40.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:40 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:18:40.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:40 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:18:40.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:40 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:18:40.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:40 smithi159 ceph-mon[93039]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-03-27T10:18:40.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:40 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:18:40.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:40 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:18:40.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:40 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:18:40.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:40 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/4006111262' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:18:40.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:40 smithi159 ceph-mon[93039]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-03-27T10:18:40.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:40 smithi159 ceph-mon[93039]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-03-27T10:18:40.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:40 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-03-27T10:18:40.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:40 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-03-27T10:18:40.617 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:40 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-03-27T10:18:40.617 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:40 smithi159 ceph-mon[93039]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-03-27T10:18:42.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:42 smithi171 ceph-mon[94808]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:42.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:42 smithi161 ceph-mon[94333]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:42.851 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:42 smithi159 ceph-mon[93039]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:44.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:44 smithi159 ceph-mon[93039]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:44.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:44 smithi171 ceph-mon[94808]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:44.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:44 smithi161 ceph-mon[94333]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:46.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:46 smithi171 ceph-mon[94808]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 44 KiB/s wr, 4 op/s 2023-03-27T10:18:46.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:46 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-03-27T10:18:46.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:46 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:46.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:46 smithi171 ceph-mon[94808]: Deploying daemon osd.3 on smithi159 2023-03-27T10:18:46.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:46 smithi161 ceph-mon[94333]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 44 KiB/s wr, 4 op/s 2023-03-27T10:18:46.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:46 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-03-27T10:18:46.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:46 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:46.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:46 smithi161 ceph-mon[94333]: Deploying daemon osd.3 on smithi159 2023-03-27T10:18:46.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:46 smithi159 ceph-mon[93039]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 44 KiB/s wr, 4 op/s 2023-03-27T10:18:46.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:46 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-03-27T10:18:46.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:46 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:46.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:46 smithi159 ceph-mon[93039]: Deploying daemon osd.3 on smithi159 2023-03-27T10:18:48.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:48 smithi171 ceph-mon[94808]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail; 1.2 KiB/s rd, 33 KiB/s wr, 3 op/s 2023-03-27T10:18:48.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:48 smithi161 ceph-mon[94333]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail; 1.2 KiB/s rd, 33 KiB/s wr, 3 op/s 2023-03-27T10:18:48.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:48 smithi159 ceph-mon[93039]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail; 1.2 KiB/s rd, 33 KiB/s wr, 3 op/s 2023-03-27T10:18:50.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:50 smithi171 ceph-mon[94808]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail; 1023 B/s rd, 26 KiB/s wr, 2 op/s 2023-03-27T10:18:50.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:50 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:50.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:50 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:50.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:50 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:50.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:50 smithi161 ceph-mon[94333]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail; 1023 B/s rd, 26 KiB/s wr, 2 op/s 2023-03-27T10:18:50.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:50 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:50.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:50 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:50.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:50 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:50 smithi159 ceph-mon[93039]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail; 1023 B/s rd, 26 KiB/s wr, 2 op/s 2023-03-27T10:18:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:50 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:50 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:50 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:52.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:52 smithi171 ceph-mon[94808]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2023-03-27T10:18:52.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:52 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:52.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:52 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:52.818 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:52 smithi159 ceph-mon[93039]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2023-03-27T10:18:52.818 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:52 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:52.819 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:52 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:52.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:52 smithi161 ceph-mon[94333]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2023-03-27T10:18:52.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:52 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:52.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:52 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:52.841 INFO:teuthology.orchestra.run.smithi159.stdout:Created osd(s) 3 on host 'smithi159' 2023-03-27T10:18:52.841 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:52.841+0000 7f7a78ff9700 1 -- 172.21.15.159:0/2202741136 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f7a7c068870 con 0x7f7a6409b350 2023-03-27T10:18:52.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:52.843+0000 7f7a81c23700 1 -- 172.21.15.159:0/2202741136 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7a6409b350 msgr2=0x7f7a6409d800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:18:52.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:52.843+0000 7f7a81c23700 1 --2- 172.21.15.159:0/2202741136 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7a6409b350 0x7f7a6409d800 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f7a6c005950 tx=0x7f7a6c00b410 comp rx=0 tx=0).stop 2023-03-27T10:18:52.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:52.843+0000 7f7a81c23700 1 -- 172.21.15.159:0/2202741136 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7a7c0feca0 msgr2=0x7f7a7c197d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:18:52.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:52.843+0000 7f7a81c23700 1 --2- 172.21.15.159:0/2202741136 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7a7c0feca0 0x7f7a7c197d30 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f7a700095c0 tx=0x7f7a7000f690 comp rx=0 tx=0).stop 2023-03-27T10:18:52.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:52.843+0000 7f7a81c23700 1 -- 172.21.15.159:0/2202741136 shutdown_connections 2023-03-27T10:18:52.844 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:52.843+0000 7f7a81c23700 1 --2- 172.21.15.159:0/2202741136 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7a6409b350 0x7f7a6409d800 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:52.844 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:52.843+0000 7f7a81c23700 1 --2- 172.21.15.159:0/2202741136 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7a7c100830 0x7f7a7c19f2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:52.844 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:52.843+0000 7f7a81c23700 1 --2- 172.21.15.159:0/2202741136 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7a7c0feca0 0x7f7a7c197d30 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:52.844 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:52.843+0000 7f7a81c23700 1 --2- 172.21.15.159:0/2202741136 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7a7c0ffea0 0x7f7a7c198270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:52.844 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:52.843+0000 7f7a81c23700 1 -- 172.21.15.159:0/2202741136 >> 172.21.15.159:0/2202741136 conn(0x7f7a7c0fa2b0 msgr2=0x7f7a7c068160 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:18:52.844 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:52.844+0000 7f7a81c23700 1 -- 172.21.15.159:0/2202741136 shutdown_connections 2023-03-27T10:18:52.844 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:18:52.844+0000 7f7a81c23700 1 -- 172.21.15.159:0/2202741136 wait complete. 2023-03-27T10:18:53.209 DEBUG:teuthology.orchestra.run.smithi159:osd.3> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.3.service 2023-03-27T10:18:53.211 INFO:tasks.cephadm:Deploying osd.4 on smithi161 with /dev/vg_nvme/lv_3... 2023-03-27T10:18:53.212 DEBUG:teuthology.orchestra.run.smithi161:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-03-27T10:18:53.241 INFO:journalctl@ceph.osd.3.smithi159.stdout:-- Logs begin at Mon 2023-03-27 10:03:12 UTC. -- 2023-03-27T10:18:53.864 INFO:teuthology.orchestra.run.smithi161.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.b/config 2023-03-27T10:18:53.940 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 27 10:18:53 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-3[111331]: 2023-03-27T10:18:53.665+0000 7f46c2976580 -1 osd.3 0 log_to_monitors true 2023-03-27T10:18:53.941 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:53 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:53.941 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:53 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:53.941 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:53 smithi159 ceph-mon[93039]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2023-03-27T10:18:53.941 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:53 smithi159 ceph-mon[93039]: from='osd.3 [v2:172.21.15.159:6810/1992276871,v1:172.21.15.159:6811/1992276871]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-03-27T10:18:54.108 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:53 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:54.109 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:53 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:54.109 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:53 smithi161 ceph-mon[94333]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2023-03-27T10:18:54.109 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:53 smithi161 ceph-mon[94333]: from='osd.3 [v2:172.21.15.159:6810/1992276871,v1:172.21.15.159:6811/1992276871]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-03-27T10:18:54.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:53 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:54.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:53 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:54.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:53 smithi171 ceph-mon[94808]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2023-03-27T10:18:54.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:53 smithi171 ceph-mon[94808]: from='osd.3 [v2:172.21.15.159:6810/1992276871,v1:172.21.15.159:6811/1992276871]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-03-27T10:18:55.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:54 smithi159 ceph-mon[93039]: from='osd.3 [v2:172.21.15.159:6810/1992276871,v1:172.21.15.159:6811/1992276871]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-03-27T10:18:55.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:54 smithi159 ceph-mon[93039]: osdmap e23: 4 total, 3 up, 4 in 2023-03-27T10:18:55.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:54 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-03-27T10:18:55.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:54 smithi159 ceph-mon[93039]: from='osd.3 [v2:172.21.15.159:6810/1992276871,v1:172.21.15.159:6811/1992276871]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2023-03-27T10:18:55.116 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 27 10:18:54 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-3[111331]: 2023-03-27T10:18:54.856+0000 7f46b7ff1700 -1 osd.3 0 waiting for initial osdmap 2023-03-27T10:18:55.117 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 27 10:18:54 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-3[111331]: 2023-03-27T10:18:54.862+0000 7f46b05df700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-03-27T10:18:55.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:54 smithi171 ceph-mon[94808]: from='osd.3 [v2:172.21.15.159:6810/1992276871,v1:172.21.15.159:6811/1992276871]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-03-27T10:18:55.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:54 smithi171 ceph-mon[94808]: osdmap e23: 4 total, 3 up, 4 in 2023-03-27T10:18:55.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:54 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-03-27T10:18:55.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:54 smithi171 ceph-mon[94808]: from='osd.3 [v2:172.21.15.159:6810/1992276871,v1:172.21.15.159:6811/1992276871]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2023-03-27T10:18:55.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:54 smithi161 ceph-mon[94333]: from='osd.3 [v2:172.21.15.159:6810/1992276871,v1:172.21.15.159:6811/1992276871]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-03-27T10:18:55.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:54 smithi161 ceph-mon[94333]: osdmap e23: 4 total, 3 up, 4 in 2023-03-27T10:18:55.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:54 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-03-27T10:18:55.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:54 smithi161 ceph-mon[94333]: from='osd.3 [v2:172.21.15.159:6810/1992276871,v1:172.21.15.159:6811/1992276871]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2023-03-27T10:18:55.504 INFO:teuthology.orchestra.run.smithi161.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-03-27T10:18:55.505 INFO:teuthology.orchestra.run.smithi161.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-03-27T10:18:55.505 INFO:teuthology.orchestra.run.smithi161.stderr: stderr: 10+0 records in 2023-03-27T10:18:55.505 INFO:teuthology.orchestra.run.smithi161.stderr:10+0 records out 2023-03-27T10:18:55.505 INFO:teuthology.orchestra.run.smithi161.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0167274 s, 627 MB/s 2023-03-27T10:18:55.505 INFO:teuthology.orchestra.run.smithi161.stderr:--> Zapping successful for: 2023-03-27T10:18:55.915 DEBUG:teuthology.orchestra.run.smithi161:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch daemon add osd smithi161:vg_nvme/lv_3 2023-03-27T10:18:56.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: from='osd.3 [v2:172.21.15.159:6810/1992276871,v1:172.21.15.159:6811/1992276871]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2023-03-27T10:18:56.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: osdmap e24: 4 total, 3 up, 4 in 2023-03-27T10:18:56.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-03-27T10:18:56.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-03-27T10:18:56.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:56.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: osd.3 [v2:172.21.15.159:6810/1992276871,v1:172.21.15.159:6811/1992276871] boot 2023-03-27T10:18:56.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: osdmap e25: 4 total, 4 up, 4 in 2023-03-27T10:18:56.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-03-27T10:18:56.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:56.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:56.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:18:56.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:18:56.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:56.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:56.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:56.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:56.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:56.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:56.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:56.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:56.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: from='osd.3 [v2:172.21.15.159:6810/1992276871,v1:172.21.15.159:6811/1992276871]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2023-03-27T10:18:56.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: osdmap e24: 4 total, 3 up, 4 in 2023-03-27T10:18:56.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-03-27T10:18:56.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-03-27T10:18:56.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:56.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: osd.3 [v2:172.21.15.159:6810/1992276871,v1:172.21.15.159:6811/1992276871] boot 2023-03-27T10:18:56.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: osdmap e25: 4 total, 4 up, 4 in 2023-03-27T10:18:56.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-03-27T10:18:56.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:56.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:56.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:18:56.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:18:56.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:56.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:56.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:56.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:56.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:56.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:56.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:56.229 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:56.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: from='osd.3 [v2:172.21.15.159:6810/1992276871,v1:172.21.15.159:6811/1992276871]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2023-03-27T10:18:56.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: osdmap e24: 4 total, 3 up, 4 in 2023-03-27T10:18:56.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-03-27T10:18:56.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-03-27T10:18:56.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-03-27T10:18:56.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: osd.3 [v2:172.21.15.159:6810/1992276871,v1:172.21.15.159:6811/1992276871] boot 2023-03-27T10:18:56.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: osdmap e25: 4 total, 4 up, 4 in 2023-03-27T10:18:56.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-03-27T10:18:56.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:56.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:56.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:18:56.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:18:56.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:56.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:56.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:56.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:56.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:18:56.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:56.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:18:56.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:18:56.549 INFO:teuthology.orchestra.run.smithi161.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.b/config 2023-03-27T10:18:57.723 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.724+0000 7f2f933fe700 1 -- 172.21.15.161:0/1876567423 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f2f8c0ffde0 msgr2=0x7f2f8c1001f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:18:57.723 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.724+0000 7f2f933fe700 1 --2- 172.21.15.161:0/1876567423 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f2f8c0ffde0 0x7f2f8c1001f0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f2f7c009990 tx=0x7f2f7c009ca0 comp rx=0 tx=0).stop 2023-03-27T10:18:57.724 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.724+0000 7f2f933fe700 1 -- 172.21.15.161:0/1876567423 shutdown_connections 2023-03-27T10:18:57.724 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.724+0000 7f2f933fe700 1 --2- 172.21.15.161:0/1876567423 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f2f8c101970 0x7f2f8c108230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:57.724 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.724+0000 7f2f933fe700 1 --2- 172.21.15.161:0/1876567423 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f2f8c0ffde0 0x7f2f8c1001f0 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:57.724 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.724+0000 7f2f933fe700 1 --2- 172.21.15.161:0/1876567423 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2f8c100fe0 0x7f2f8c101430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:57.724 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.724+0000 7f2f933fe700 1 -- 172.21.15.161:0/1876567423 >> 172.21.15.161:0/1876567423 conn(0x7f2f8c0fb450 msgr2=0x7f2f8c0fd860 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:18:57.724 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.724+0000 7f2f933fe700 1 -- 172.21.15.161:0/1876567423 shutdown_connections 2023-03-27T10:18:57.724 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.724+0000 7f2f933fe700 1 -- 172.21.15.161:0/1876567423 wait complete. 2023-03-27T10:18:57.726 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f933fe700 1 Processor -- start 2023-03-27T10:18:57.726 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f933fe700 1 -- start start 2023-03-27T10:18:57.726 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f933fe700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f2f8c0ffde0 0x7f2f8c195b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:18:57.726 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f933fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2f8c100fe0 0x7f2f8c196050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:18:57.727 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f933fe700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f2f8c101970 0x7f2f8c19d0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:18:57.727 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f933fe700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f8c19d6a0 con 0x7f2f8c100fe0 2023-03-27T10:18:57.727 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f933fe700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f8c19d7e0 con 0x7f2f8c0ffde0 2023-03-27T10:18:57.727 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f933fe700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f8c19d970 con 0x7f2f8c101970 2023-03-27T10:18:57.727 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f9119a700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f2f8c0ffde0 0x7f2f8c195b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:18:57.727 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f9119a700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f2f8c0ffde0 0x7f2f8c195b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.161:3300/0 says I am v2:172.21.15.161:44564/0 (socket says 172.21.15.161:44564) 2023-03-27T10:18:57.727 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f9119a700 1 -- 172.21.15.161:0/3743034530 learned_addr learned my addr 172.21.15.161:0/3743034530 (peer_addr_for_me v2:172.21.15.161:0/0) 2023-03-27T10:18:57.727 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f90999700 1 --2- 172.21.15.161:0/3743034530 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2f8c100fe0 0x7f2f8c196050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:18:57.728 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f9199b700 1 --2- 172.21.15.161:0/3743034530 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f2f8c101970 0x7f2f8c19d0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:18:57.728 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f90999700 1 -- 172.21.15.161:0/3743034530 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f2f8c101970 msgr2=0x7f2f8c19d0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:18:57.728 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f90999700 1 --2- 172.21.15.161:0/3743034530 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f2f8c101970 0x7f2f8c19d0d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:57.728 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f90999700 1 -- 172.21.15.161:0/3743034530 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f2f8c0ffde0 msgr2=0x7f2f8c195b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:18:57.728 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f90999700 1 --2- 172.21.15.161:0/3743034530 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f2f8c0ffde0 0x7f2f8c195b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:18:57.728 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f90999700 1 -- 172.21.15.161:0/3743034530 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2f7c0095f0 con 0x7f2f8c100fe0 2023-03-27T10:18:57.728 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.725+0000 7f2f9119a700 1 --2- 172.21.15.161:0/3743034530 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f2f8c0ffde0 0x7f2f8c195b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-03-27T10:18:57.729 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.726+0000 7f2f90999700 1 --2- 172.21.15.161:0/3743034530 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2f8c100fe0 0x7f2f8c196050 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f2f8800ea20 tx=0x7f2f8800ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:18:57.729 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.726+0000 7f2f827fc700 1 -- 172.21.15.161:0/3743034530 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f8800cbd0 con 0x7f2f8c100fe0 2023-03-27T10:18:57.729 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.726+0000 7f2f933fe700 1 -- 172.21.15.161:0/3743034530 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2f8c10b730 con 0x7f2f8c100fe0 2023-03-27T10:18:57.729 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.726+0000 7f2f827fc700 1 -- 172.21.15.161:0/3743034530 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2f8800ee60 con 0x7f2f8c100fe0 2023-03-27T10:18:57.729 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.726+0000 7f2f827fc700 1 -- 172.21.15.161:0/3743034530 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f88018680 con 0x7f2f8c100fe0 2023-03-27T10:18:57.729 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.726+0000 7f2f933fe700 1 -- 172.21.15.161:0/3743034530 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2f8c10b5a0 con 0x7f2f8c100fe0 2023-03-27T10:18:57.730 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.727+0000 7f2f933fe700 1 -- 172.21.15.161:0/3743034530 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2f8c107420 con 0x7f2f8c100fe0 2023-03-27T10:18:57.730 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:57 smithi171 ceph-mon[94808]: purged_snaps scrub starts 2023-03-27T10:18:57.730 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:57 smithi171 ceph-mon[94808]: purged_snaps scrub ok 2023-03-27T10:18:57.730 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:57 smithi171 ceph-mon[94808]: Detected new or changed devices on smithi159 2023-03-27T10:18:57.731 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:57 smithi171 ceph-mon[94808]: Adjusting osd_memory_target on smithi159 to 8584M 2023-03-27T10:18:57.731 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:57 smithi171 ceph-mon[94808]: osdmap e26: 4 total, 4 up, 4 in 2023-03-27T10:18:57.731 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.728+0000 7f2f827fc700 1 -- 172.21.15.161:0/3743034530 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f2f8800c040 con 0x7f2f8c100fe0 2023-03-27T10:18:57.732 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.729+0000 7f2f827fc700 1 --2- 172.21.15.161:0/3743034530 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f2f7809b3a0 0x7f2f7809d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:18:57.732 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.729+0000 7f2f827fc700 1 -- 172.21.15.161:0/3743034530 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(27..27 src has 1..27) v4 ==== 3698+0+0 (secure 0 0 0) 0x7f2f88010e40 con 0x7f2f8c100fe0 2023-03-27T10:18:57.732 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.729+0000 7f2f9119a700 1 --2- 172.21.15.161:0/3743034530 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f2f7809b3a0 0x7f2f7809d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:18:57.732 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.729+0000 7f2f9119a700 1 --2- 172.21.15.161:0/3743034530 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f2f7809b3a0 0x7f2f7809d850 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f2f7c00b680 tx=0x7f2f7c005fd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:18:57.732 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.732+0000 7f2f827fc700 1 -- 172.21.15.161:0/3743034530 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f2f88090cb0 con 0x7f2f8c100fe0 2023-03-27T10:18:57.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:57 smithi161 ceph-mon[94333]: purged_snaps scrub starts 2023-03-27T10:18:57.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:57 smithi161 ceph-mon[94333]: purged_snaps scrub ok 2023-03-27T10:18:57.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:57 smithi161 ceph-mon[94333]: Detected new or changed devices on smithi159 2023-03-27T10:18:57.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:57 smithi161 ceph-mon[94333]: Adjusting osd_memory_target on smithi159 to 8584M 2023-03-27T10:18:57.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:57 smithi161 ceph-mon[94333]: osdmap e26: 4 total, 4 up, 4 in 2023-03-27T10:18:57.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:57 smithi159 ceph-mon[93039]: purged_snaps scrub starts 2023-03-27T10:18:57.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:57 smithi159 ceph-mon[93039]: purged_snaps scrub ok 2023-03-27T10:18:57.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:57 smithi159 ceph-mon[93039]: Detected new or changed devices on smithi159 2023-03-27T10:18:57.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:57 smithi159 ceph-mon[93039]: Adjusting osd_memory_target on smithi159 to 8584M 2023-03-27T10:18:57.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:57 smithi159 ceph-mon[93039]: osdmap e26: 4 total, 4 up, 4 in 2023-03-27T10:18:57.921 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:18:57.922+0000 7f2f933fe700 1 -- 172.21.15.161:0/3743034530 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi161:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) v1 -- 0x7f2f8c0618a0 con 0x7f2f7809b3a0 2023-03-27T10:18:58.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:58 smithi171 ceph-mon[94808]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 22 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:18:58.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:58 smithi171 ceph-mon[94808]: osdmap e27: 4 total, 4 up, 4 in 2023-03-27T10:18:58.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:58 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:18:58.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:58 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:18:58.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:58 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:58.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:58 smithi161 ceph-mon[94333]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 22 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:18:58.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:58 smithi161 ceph-mon[94333]: osdmap e27: 4 total, 4 up, 4 in 2023-03-27T10:18:58.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:58 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:18:58.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:58 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:18:58.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:58 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:58.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:58 smithi159 ceph-mon[93039]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 22 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:18:58.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:58 smithi159 ceph-mon[93039]: osdmap e27: 4 total, 4 up, 4 in 2023-03-27T10:18:58.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:58 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:18:58.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:58 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:18:58.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:58 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:18:59.461 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:59 smithi161 ceph-mon[94333]: from='client.14325 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi161:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:18:59.462 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:18:59 smithi161 ceph-mon[94333]: osdmap e28: 4 total, 4 up, 4 in 2023-03-27T10:18:59.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:59 smithi171 ceph-mon[94808]: from='client.14325 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi161:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:18:59.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:18:59 smithi171 ceph-mon[94808]: osdmap e28: 4 total, 4 up, 4 in 2023-03-27T10:18:59.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:59 smithi159 ceph-mon[93039]: from='client.14325 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi161:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:18:59.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:18:59 smithi159 ceph-mon[93039]: osdmap e28: 4 total, 4 up, 4 in 2023-03-27T10:19:00.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:00 smithi171 ceph-mon[94808]: pgmap v94: 1 pgs: 1 peering; 577 KiB data, 22 MiB used, 358 GiB / 358 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-03-27T10:19:00.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:00 smithi171 ceph-mon[94808]: osdmap e29: 4 total, 4 up, 4 in 2023-03-27T10:19:00.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:00 smithi161 ceph-mon[94333]: pgmap v94: 1 pgs: 1 peering; 577 KiB data, 22 MiB used, 358 GiB / 358 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-03-27T10:19:00.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:00 smithi161 ceph-mon[94333]: osdmap e29: 4 total, 4 up, 4 in 2023-03-27T10:19:00.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:00 smithi159 ceph-mon[93039]: pgmap v94: 1 pgs: 1 peering; 577 KiB data, 22 MiB used, 358 GiB / 358 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-03-27T10:19:00.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:00 smithi159 ceph-mon[93039]: osdmap e29: 4 total, 4 up, 4 in 2023-03-27T10:19:01.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:01 smithi171 ceph-mon[94808]: from='client.? 172.21.15.161:0/131911456' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "982675c1-dd87-46b4-97ef-a5dde1b1d777"}]: dispatch 2023-03-27T10:19:01.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:01 smithi171 ceph-mon[94808]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "982675c1-dd87-46b4-97ef-a5dde1b1d777"}]: dispatch 2023-03-27T10:19:01.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:01 smithi171 ceph-mon[94808]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "982675c1-dd87-46b4-97ef-a5dde1b1d777"}]': finished 2023-03-27T10:19:01.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:01 smithi171 ceph-mon[94808]: osdmap e30: 5 total, 4 up, 5 in 2023-03-27T10:19:01.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:01 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-03-27T10:19:01.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:01 smithi171 ceph-mon[94808]: from='client.? 172.21.15.161:0/3531790043' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:19:01.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:01 smithi161 ceph-mon[94333]: from='client.? 172.21.15.161:0/131911456' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "982675c1-dd87-46b4-97ef-a5dde1b1d777"}]: dispatch 2023-03-27T10:19:01.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:01 smithi161 ceph-mon[94333]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "982675c1-dd87-46b4-97ef-a5dde1b1d777"}]: dispatch 2023-03-27T10:19:01.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:01 smithi161 ceph-mon[94333]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "982675c1-dd87-46b4-97ef-a5dde1b1d777"}]': finished 2023-03-27T10:19:01.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:01 smithi161 ceph-mon[94333]: osdmap e30: 5 total, 4 up, 5 in 2023-03-27T10:19:01.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:01 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-03-27T10:19:01.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:01 smithi161 ceph-mon[94333]: from='client.? 172.21.15.161:0/3531790043' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:19:01.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:01 smithi159 ceph-mon[93039]: from='client.? 172.21.15.161:0/131911456' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "982675c1-dd87-46b4-97ef-a5dde1b1d777"}]: dispatch 2023-03-27T10:19:01.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:01 smithi159 ceph-mon[93039]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "982675c1-dd87-46b4-97ef-a5dde1b1d777"}]: dispatch 2023-03-27T10:19:01.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:01 smithi159 ceph-mon[93039]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "982675c1-dd87-46b4-97ef-a5dde1b1d777"}]': finished 2023-03-27T10:19:01.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:01 smithi159 ceph-mon[93039]: osdmap e30: 5 total, 4 up, 5 in 2023-03-27T10:19:01.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:01 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-03-27T10:19:01.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:01 smithi159 ceph-mon[93039]: from='client.? 172.21.15.161:0/3531790043' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:19:02.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:02 smithi171 ceph-mon[94808]: pgmap v97: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-03-27T10:19:02.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:02 smithi161 ceph-mon[94333]: pgmap v97: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-03-27T10:19:02.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:02 smithi159 ceph-mon[93039]: pgmap v97: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-03-27T10:19:04.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:04 smithi171 ceph-mon[94808]: pgmap v98: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-03-27T10:19:04.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:04 smithi161 ceph-mon[94333]: pgmap v98: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-03-27T10:19:04.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:04 smithi159 ceph-mon[93039]: pgmap v98: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-03-27T10:19:06.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:06 smithi161 ceph-mon[94333]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-03-27T10:19:06.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:06 smithi171 ceph-mon[94808]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-03-27T10:19:06.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:06 smithi159 ceph-mon[93039]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-03-27T10:19:07.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:07 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-03-27T10:19:07.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:07 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:07.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:07 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-03-27T10:19:07.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:07 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:07.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:07 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-03-27T10:19:07.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:07 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:08.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:08 smithi161 ceph-mon[94333]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:08.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:08 smithi161 ceph-mon[94333]: Deploying daemon osd.4 on smithi161 2023-03-27T10:19:08.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:08 smithi171 ceph-mon[94808]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:08.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:08 smithi171 ceph-mon[94808]: Deploying daemon osd.4 on smithi161 2023-03-27T10:19:08.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:08 smithi159 ceph-mon[93039]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:08.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:08 smithi159 ceph-mon[93039]: Deploying daemon osd.4 on smithi161 2023-03-27T10:19:10.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:10 smithi171 ceph-mon[94808]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:10.808 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:10 smithi161 ceph-mon[94333]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:10.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:10 smithi159 ceph-mon[93039]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:12.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:12 smithi171 ceph-mon[94808]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:12.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:12 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:12.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:12 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:12.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:12 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:19:12.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:12 smithi161 ceph-mon[94333]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:12.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:12 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:12.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:12 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:12.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:12 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:19:12.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:12 smithi159 ceph-mon[93039]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:12.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:12 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:12.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:12 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:12.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:12 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:19:14.577 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:14 smithi161 ceph-mon[94333]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:14.577 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:14 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:14.577 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:14 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:14.672 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:19:14.672+0000 7f2f827fc700 1 -- 172.21.15.161:0/3743034530 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f2f8c0618a0 con 0x7f2f7809b3a0 2023-03-27T10:19:14.672 INFO:teuthology.orchestra.run.smithi161.stdout:Created osd(s) 4 on host 'smithi161' 2023-03-27T10:19:14.674 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:19:14.674+0000 7f2f933fe700 1 -- 172.21.15.161:0/3743034530 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f2f7809b3a0 msgr2=0x7f2f7809d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:19:14.674 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:19:14.674+0000 7f2f933fe700 1 --2- 172.21.15.161:0/3743034530 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f2f7809b3a0 0x7f2f7809d850 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f2f7c00b680 tx=0x7f2f7c005fd0 comp rx=0 tx=0).stop 2023-03-27T10:19:14.674 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:19:14.674+0000 7f2f933fe700 1 -- 172.21.15.161:0/3743034530 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2f8c100fe0 msgr2=0x7f2f8c196050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:19:14.674 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:19:14.674+0000 7f2f933fe700 1 --2- 172.21.15.161:0/3743034530 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2f8c100fe0 0x7f2f8c196050 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f2f8800ea20 tx=0x7f2f8800ed30 comp rx=0 tx=0).stop 2023-03-27T10:19:14.674 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:19:14.675+0000 7f2f933fe700 1 -- 172.21.15.161:0/3743034530 shutdown_connections 2023-03-27T10:19:14.674 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:19:14.675+0000 7f2f933fe700 1 --2- 172.21.15.161:0/3743034530 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f2f7809b3a0 0x7f2f7809d850 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:14.674 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:19:14.675+0000 7f2f933fe700 1 --2- 172.21.15.161:0/3743034530 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f2f8c101970 0x7f2f8c19d0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:14.675 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:19:14.675+0000 7f2f933fe700 1 --2- 172.21.15.161:0/3743034530 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f2f8c0ffde0 0x7f2f8c195b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:14.675 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:19:14.675+0000 7f2f933fe700 1 --2- 172.21.15.161:0/3743034530 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2f8c100fe0 0x7f2f8c196050 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:14.675 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:19:14.675+0000 7f2f933fe700 1 -- 172.21.15.161:0/3743034530 >> 172.21.15.161:0/3743034530 conn(0x7f2f8c0fb450 msgr2=0x7f2f8c104560 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:19:14.675 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:19:14.675+0000 7f2f933fe700 1 -- 172.21.15.161:0/3743034530 shutdown_connections 2023-03-27T10:19:14.675 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:19:14.675+0000 7f2f933fe700 1 -- 172.21.15.161:0/3743034530 wait complete. 2023-03-27T10:19:14.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:14 smithi171 ceph-mon[94808]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:14.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:14 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:14.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:14 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:14.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:14 smithi159 ceph-mon[93039]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:14.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:14 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:14.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:14 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:15.139 DEBUG:teuthology.orchestra.run.smithi161:osd.4> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.4.service 2023-03-27T10:19:15.141 INFO:tasks.cephadm:Deploying osd.5 on smithi171 with /dev/vg_nvme/lv_3... 2023-03-27T10:19:15.142 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-03-27T10:19:15.171 INFO:journalctl@ceph.osd.4.smithi161.stdout:-- Logs begin at Mon 2023-03-27 10:02:59 UTC. -- 2023-03-27T10:19:15.792 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.c/config 2023-03-27T10:19:15.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:15 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:15.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:15 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:15.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:15 smithi161 ceph-mon[94333]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:15.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:15 smithi161 ceph-mon[94333]: from='osd.4 [v2:172.21.15.161:6808/3978602555,v1:172.21.15.161:6809/3978602555]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-03-27T10:19:15.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:15 smithi161 ceph-mon[94333]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-03-27T10:19:15.834 INFO:journalctl@ceph.osd.4.smithi161.stdout:Mar 27 10:19:15 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-4[104810]: 2023-03-27T10:19:15.554+0000 7f5589f0c580 -1 osd.4 0 log_to_monitors true 2023-03-27T10:19:15.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:15 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:15.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:15 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:15.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:15 smithi171 ceph-mon[94808]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:15.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:15 smithi171 ceph-mon[94808]: from='osd.4 [v2:172.21.15.161:6808/3978602555,v1:172.21.15.161:6809/3978602555]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-03-27T10:19:15.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:15 smithi171 ceph-mon[94808]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-03-27T10:19:16.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:15 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:16.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:15 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:16.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:15 smithi159 ceph-mon[93039]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:16.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:15 smithi159 ceph-mon[93039]: from='osd.4 [v2:172.21.15.161:6808/3978602555,v1:172.21.15.161:6809/3978602555]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-03-27T10:19:16.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:15 smithi159 ceph-mon[93039]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-03-27T10:19:17.341 INFO:teuthology.orchestra.run.smithi171.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-03-27T10:19:17.342 INFO:teuthology.orchestra.run.smithi171.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-03-27T10:19:17.342 INFO:teuthology.orchestra.run.smithi171.stderr: stderr: 10+0 records in 2023-03-27T10:19:17.342 INFO:teuthology.orchestra.run.smithi171.stderr:10+0 records out 2023-03-27T10:19:17.342 INFO:teuthology.orchestra.run.smithi171.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0110935 s, 945 MB/s 2023-03-27T10:19:17.342 INFO:teuthology.orchestra.run.smithi171.stderr:--> Zapping successful for: 2023-03-27T10:19:17.583 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:17 smithi161 ceph-mon[94333]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-03-27T10:19:17.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:17 smithi161 ceph-mon[94333]: osdmap e31: 5 total, 4 up, 5 in 2023-03-27T10:19:17.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:17 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-03-27T10:19:17.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:17 smithi161 ceph-mon[94333]: from='osd.4 [v2:172.21.15.161:6808/3978602555,v1:172.21.15.161:6809/3978602555]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:19:17.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:17 smithi161 ceph-mon[94333]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:19:17.584 INFO:journalctl@ceph.osd.4.smithi161.stdout:Mar 27 10:19:17 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-4[104810]: 2023-03-27T10:19:17.402+0000 7f557f587700 -1 osd.4 0 waiting for initial osdmap 2023-03-27T10:19:17.585 INFO:journalctl@ceph.osd.4.smithi161.stdout:Mar 27 10:19:17 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-4[104810]: 2023-03-27T10:19:17.408+0000 7f557b37c700 -1 osd.4 32 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-03-27T10:19:17.720 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:17 smithi171 ceph-mon[94808]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-03-27T10:19:17.721 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:17 smithi171 ceph-mon[94808]: osdmap e31: 5 total, 4 up, 5 in 2023-03-27T10:19:17.721 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:17 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-03-27T10:19:17.721 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:17 smithi171 ceph-mon[94808]: from='osd.4 [v2:172.21.15.161:6808/3978602555,v1:172.21.15.161:6809/3978602555]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:19:17.721 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:17 smithi171 ceph-mon[94808]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:19:17.750 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch daemon add osd smithi171:vg_nvme/lv_3 2023-03-27T10:19:17.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:17 smithi159 ceph-mon[93039]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-03-27T10:19:17.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:17 smithi159 ceph-mon[93039]: osdmap e31: 5 total, 4 up, 5 in 2023-03-27T10:19:17.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:17 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-03-27T10:19:17.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:17 smithi159 ceph-mon[93039]: from='osd.4 [v2:172.21.15.161:6808/3978602555,v1:172.21.15.161:6809/3978602555]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:19:17.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:17 smithi159 ceph-mon[93039]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:19:18.391 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.c/config 2023-03-27T10:19:18.668 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:18.668 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]': finished 2023-03-27T10:19:18.668 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: osdmap e32: 5 total, 4 up, 5 in 2023-03-27T10:19:18.668 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-03-27T10:19:18.669 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-03-27T10:19:18.669 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:18.669 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:18.669 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:19:18.669 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:19:18.669 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:18.669 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:18.669 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:19:18.670 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:18.670 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:19:18.670 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:18.670 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:19:18.670 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:18 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:18.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:18.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]': finished 2023-03-27T10:19:18.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: osdmap e32: 5 total, 4 up, 5 in 2023-03-27T10:19:18.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-03-27T10:19:18.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-03-27T10:19:18.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:18.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:18.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:19:18.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:19:18.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:18.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:18.836 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:19:18.836 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:18.836 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:19:18.836 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:18.836 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:19:18.836 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:18 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:18.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:18.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]': finished 2023-03-27T10:19:18.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: osdmap e32: 5 total, 4 up, 5 in 2023-03-27T10:19:18.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-03-27T10:19:18.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-03-27T10:19:18.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:18.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:18.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:19:18.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:19:18.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:18.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:18.867 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:19:18.867 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:18.867 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:19:18.867 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:18.867 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:19:18.867 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:18 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:19.609 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.609+0000 7f79136b8700 1 -- 172.21.15.171:0/3353255327 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f790c101a90 msgr2=0x7f790c108350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:19:19.609 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.609+0000 7f79136b8700 1 --2- 172.21.15.171:0/3353255327 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f790c101a90 0x7f790c108350 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f790400b0a0 tx=0x7f790400b3b0 comp rx=0 tx=0).stop 2023-03-27T10:19:19.610 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.609+0000 7f79136b8700 1 -- 172.21.15.171:0/3353255327 shutdown_connections 2023-03-27T10:19:19.610 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.609+0000 7f79136b8700 1 --2- 172.21.15.171:0/3353255327 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f790c101a90 0x7f790c108350 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:19.610 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.609+0000 7f79136b8700 1 --2- 172.21.15.171:0/3353255327 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f790c0fe080 0x7f790c0fe490 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:19.610 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.609+0000 7f79136b8700 1 --2- 172.21.15.171:0/3353255327 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f790c101100 0x7f790c101550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:19.610 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.609+0000 7f79136b8700 1 -- 172.21.15.171:0/3353255327 >> 172.21.15.171:0/3353255327 conn(0x7f790c0f9a30 msgr2=0x7f790c0fbea0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:19:19.610 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.609+0000 7f79136b8700 1 -- 172.21.15.171:0/3353255327 shutdown_connections 2023-03-27T10:19:19.610 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.609+0000 7f79136b8700 1 -- 172.21.15.171:0/3353255327 wait complete. 2023-03-27T10:19:19.612 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f79136b8700 1 Processor -- start 2023-03-27T10:19:19.612 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f79136b8700 1 -- start start 2023-03-27T10:19:19.612 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f79136b8700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f790c0fe080 0x7f790c195bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:19:19.613 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f79136b8700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f790c101100 0x7f790c196100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:19:19.613 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f79136b8700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f790c101a90 0x7f790c19d180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:19:19.613 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f79136b8700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f790c19d750 con 0x7f790c0fe080 2023-03-27T10:19:19.613 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f79136b8700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f790c19d890 con 0x7f790c101a90 2023-03-27T10:19:19.613 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f79136b8700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f790c19da20 con 0x7f790c101100 2023-03-27T10:19:19.613 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f7910c53700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f790c101100 0x7f790c196100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:19:19.614 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f7910c53700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f790c101100 0x7f790c196100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.171:3300/0 says I am v2:172.21.15.171:55212/0 (socket says 172.21.15.171:55212) 2023-03-27T10:19:19.614 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f7910c53700 1 -- 172.21.15.171:0/1011055164 learned_addr learned my addr 172.21.15.171:0/1011055164 (peer_addr_for_me v2:172.21.15.171:0/0) 2023-03-27T10:19:19.614 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f7911454700 1 --2- 172.21.15.171:0/1011055164 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f790c0fe080 0x7f790c195bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:19:19.614 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f7911c55700 1 --2- 172.21.15.171:0/1011055164 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f790c101a90 0x7f790c19d180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:19:19.614 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f7910c53700 1 -- 172.21.15.171:0/1011055164 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f790c101a90 msgr2=0x7f790c19d180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:19:19.614 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f7910c53700 1 --2- 172.21.15.171:0/1011055164 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f790c101a90 0x7f790c19d180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:19.614 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f7910c53700 1 -- 172.21.15.171:0/1011055164 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f790c0fe080 msgr2=0x7f790c195bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:19:19.614 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f7910c53700 1 --2- 172.21.15.171:0/1011055164 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f790c0fe080 0x7f790c195bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:19.615 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f7910c53700 1 -- 172.21.15.171:0/1011055164 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7904009d00 con 0x7f790c101100 2023-03-27T10:19:19.615 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.610+0000 7f7911c55700 1 --2- 172.21.15.171:0/1011055164 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f790c101a90 0x7f790c19d180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:19:19.615 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.611+0000 7f7910c53700 1 --2- 172.21.15.171:0/1011055164 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f790c101100 0x7f790c196100 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f790800d980 tx=0x7f790800dc90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:19:19.615 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.611+0000 7f79027fc700 1 -- 172.21.15.171:0/1011055164 <== mon.1 v2:172.21.15.171:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7908009b40 con 0x7f790c101100 2023-03-27T10:19:19.615 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.611+0000 7f79136b8700 1 -- 172.21.15.171:0/1011055164 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f790c10b710 con 0x7f790c101100 2023-03-27T10:19:19.615 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.611+0000 7f79027fc700 1 -- 172.21.15.171:0/1011055164 <== mon.1 v2:172.21.15.171:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f790800b6e0 con 0x7f790c101100 2023-03-27T10:19:19.615 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.611+0000 7f79027fc700 1 -- 172.21.15.171:0/1011055164 <== mon.1 v2:172.21.15.171:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f790800f620 con 0x7f790c101100 2023-03-27T10:19:19.616 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.611+0000 7f79136b8700 1 -- 172.21.15.171:0/1011055164 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f790c10b580 con 0x7f790c101100 2023-03-27T10:19:19.616 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.612+0000 7f79136b8700 1 -- 172.21.15.171:0/1011055164 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f790c1073f0 con 0x7f790c101100 2023-03-27T10:19:19.616 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.613+0000 7f79027fc700 1 -- 172.21.15.171:0/1011055164 <== mon.1 v2:172.21.15.171:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f7908018030 con 0x7f790c101100 2023-03-27T10:19:19.616 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.613+0000 7f79027fc700 1 --2- 172.21.15.171:0/1011055164 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f78f809b3a0 0x7f78f809d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:19:19.617 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.613+0000 7f79027fc700 1 -- 172.21.15.171:0/1011055164 <== mon.1 v2:172.21.15.171:3300/0 5 ==== osd_map(34..34 src has 1..34) v4 ==== 4101+0+0 (secure 0 0 0) 0x7f79080c4000 con 0x7f790c101100 2023-03-27T10:19:19.617 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.613+0000 7f7911454700 1 --2- 172.21.15.171:0/1011055164 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f78f809b3a0 0x7f78f809d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:19:19.617 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.614+0000 7f7911454700 1 --2- 172.21.15.171:0/1011055164 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f78f809b3a0 0x7f78f809d850 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f790c067de0 tx=0x7f78fc00b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:19:19.617 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.617+0000 7f79027fc700 1 -- 172.21.15.171:0/1011055164 <== mon.1 v2:172.21.15.171:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f79080909e0 con 0x7f790c101100 2023-03-27T10:19:19.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:19 smithi171 ceph-mon[94808]: purged_snaps scrub starts 2023-03-27T10:19:19.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:19 smithi171 ceph-mon[94808]: purged_snaps scrub ok 2023-03-27T10:19:19.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:19 smithi171 ceph-mon[94808]: Detected new or changed devices on smithi161 2023-03-27T10:19:19.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:19 smithi171 ceph-mon[94808]: Adjusting osd_memory_target on smithi161 to 8584M 2023-03-27T10:19:19.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:19 smithi171 ceph-mon[94808]: osd.4 [v2:172.21.15.161:6808/3978602555,v1:172.21.15.161:6809/3978602555] boot 2023-03-27T10:19:19.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:19 smithi171 ceph-mon[94808]: osdmap e33: 5 total, 5 up, 5 in 2023-03-27T10:19:19.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:19 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-03-27T10:19:19.808 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:19.808+0000 7f79136b8700 1 -- 172.21.15.171:0/1011055164 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) v1 -- 0x7f790c061990 con 0x7f78f809b3a0 2023-03-27T10:19:19.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:19 smithi161 ceph-mon[94333]: purged_snaps scrub starts 2023-03-27T10:19:19.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:19 smithi161 ceph-mon[94333]: purged_snaps scrub ok 2023-03-27T10:19:19.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:19 smithi161 ceph-mon[94333]: Detected new or changed devices on smithi161 2023-03-27T10:19:19.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:19 smithi161 ceph-mon[94333]: Adjusting osd_memory_target on smithi161 to 8584M 2023-03-27T10:19:19.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:19 smithi161 ceph-mon[94333]: osd.4 [v2:172.21.15.161:6808/3978602555,v1:172.21.15.161:6809/3978602555] boot 2023-03-27T10:19:19.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:19 smithi161 ceph-mon[94333]: osdmap e33: 5 total, 5 up, 5 in 2023-03-27T10:19:19.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:19 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-03-27T10:19:19.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:19 smithi159 ceph-mon[93039]: purged_snaps scrub starts 2023-03-27T10:19:19.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:19 smithi159 ceph-mon[93039]: purged_snaps scrub ok 2023-03-27T10:19:19.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:19 smithi159 ceph-mon[93039]: Detected new or changed devices on smithi161 2023-03-27T10:19:19.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:19 smithi159 ceph-mon[93039]: Adjusting osd_memory_target on smithi161 to 8584M 2023-03-27T10:19:19.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:19 smithi159 ceph-mon[93039]: osd.4 [v2:172.21.15.161:6808/3978602555,v1:172.21.15.161:6809/3978602555] boot 2023-03-27T10:19:19.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:19 smithi159 ceph-mon[93039]: osdmap e33: 5 total, 5 up, 5 in 2023-03-27T10:19:19.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:19 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-03-27T10:19:20.690 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:20 smithi171 ceph-mon[94808]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:20.690 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:20 smithi171 ceph-mon[94808]: osdmap e34: 5 total, 5 up, 5 in 2023-03-27T10:19:20.690 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:20 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:19:20.690 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:20 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:19:20.691 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:20 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:20.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:20 smithi161 ceph-mon[94333]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:20.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:20 smithi161 ceph-mon[94333]: osdmap e34: 5 total, 5 up, 5 in 2023-03-27T10:19:20.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:20 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:19:20.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:20 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:19:20.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:20 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:20.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:20 smithi159 ceph-mon[93039]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-03-27T10:19:20.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:20 smithi159 ceph-mon[93039]: osdmap e34: 5 total, 5 up, 5 in 2023-03-27T10:19:20.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:20 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:19:20.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:20 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:19:20.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:20 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:21.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:21 smithi171 ceph-mon[94808]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:19:21.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:21 smithi161 ceph-mon[94333]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:19:21.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:21 smithi159 ceph-mon[93039]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:19:22.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:22 smithi171 ceph-mon[94808]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:22.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:22 smithi171 ceph-mon[94808]: from='client.? 172.21.15.171:0/3849869987' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4de4a85-4f91-4194-a1ce-db12f1898c4f"}]: dispatch 2023-03-27T10:19:22.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:22 smithi171 ceph-mon[94808]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4de4a85-4f91-4194-a1ce-db12f1898c4f"}]: dispatch 2023-03-27T10:19:22.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:22 smithi171 ceph-mon[94808]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b4de4a85-4f91-4194-a1ce-db12f1898c4f"}]': finished 2023-03-27T10:19:22.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:22 smithi171 ceph-mon[94808]: osdmap e35: 6 total, 5 up, 6 in 2023-03-27T10:19:22.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:22 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-03-27T10:19:22.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:22 smithi161 ceph-mon[94333]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:22.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:22 smithi161 ceph-mon[94333]: from='client.? 172.21.15.171:0/3849869987' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4de4a85-4f91-4194-a1ce-db12f1898c4f"}]: dispatch 2023-03-27T10:19:22.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:22 smithi161 ceph-mon[94333]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4de4a85-4f91-4194-a1ce-db12f1898c4f"}]: dispatch 2023-03-27T10:19:22.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:22 smithi161 ceph-mon[94333]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b4de4a85-4f91-4194-a1ce-db12f1898c4f"}]': finished 2023-03-27T10:19:22.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:22 smithi161 ceph-mon[94333]: osdmap e35: 6 total, 5 up, 6 in 2023-03-27T10:19:22.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:22 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-03-27T10:19:22.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:22 smithi159 ceph-mon[93039]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:22.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:22 smithi159 ceph-mon[93039]: from='client.? 172.21.15.171:0/3849869987' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4de4a85-4f91-4194-a1ce-db12f1898c4f"}]: dispatch 2023-03-27T10:19:22.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:22 smithi159 ceph-mon[93039]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4de4a85-4f91-4194-a1ce-db12f1898c4f"}]: dispatch 2023-03-27T10:19:22.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:22 smithi159 ceph-mon[93039]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b4de4a85-4f91-4194-a1ce-db12f1898c4f"}]': finished 2023-03-27T10:19:22.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:22 smithi159 ceph-mon[93039]: osdmap e35: 6 total, 5 up, 6 in 2023-03-27T10:19:22.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:22 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-03-27T10:19:23.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:23 smithi171 ceph-mon[94808]: from='client.? 172.21.15.171:0/1526600591' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:19:23.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:23 smithi161 ceph-mon[94333]: from='client.? 172.21.15.171:0/1526600591' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:19:23.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:23 smithi159 ceph-mon[93039]: from='client.? 172.21.15.171:0/1526600591' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:19:24.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:24 smithi171 ceph-mon[94808]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:24.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:24 smithi161 ceph-mon[94333]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:24.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:24 smithi159 ceph-mon[93039]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:26.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:26 smithi171 ceph-mon[94808]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:26.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:26 smithi161 ceph-mon[94333]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:26.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:26 smithi159 ceph-mon[93039]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:28.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:28 smithi171 ceph-mon[94808]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:28.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:28 smithi161 ceph-mon[94333]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:28.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:28 smithi159 ceph-mon[93039]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:29.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:29 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-03-27T10:19:29.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:29 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:29.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:29 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-03-27T10:19:29.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:29 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:29.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:29 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-03-27T10:19:29.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:29 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:30.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:30 smithi171 ceph-mon[94808]: Deploying daemon osd.5 on smithi171 2023-03-27T10:19:30.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:30 smithi171 ceph-mon[94808]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:30.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:30 smithi161 ceph-mon[94333]: Deploying daemon osd.5 on smithi171 2023-03-27T10:19:30.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:30 smithi161 ceph-mon[94333]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:30.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:30 smithi159 ceph-mon[93039]: Deploying daemon osd.5 on smithi171 2023-03-27T10:19:30.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:30 smithi159 ceph-mon[93039]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:32.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:32 smithi171 ceph-mon[94808]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:32.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:32 smithi161 ceph-mon[94333]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:32.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:32 smithi159 ceph-mon[93039]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:34.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:34 smithi171 ceph-mon[94808]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:34.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:34 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:34.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:34 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:34.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:34 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:19:34.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:34 smithi161 ceph-mon[94333]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:34.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:34 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:34.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:34 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:34.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:34 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:19:34.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:34 smithi159 ceph-mon[93039]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:34.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:34 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:34.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:34 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:34.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:34 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:19:36.452 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:36.453+0000 7f79027fc700 1 -- 172.21.15.171:0/1011055164 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f790c061990 con 0x7f78f809b3a0 2023-03-27T10:19:36.452 INFO:teuthology.orchestra.run.smithi171.stdout:Created osd(s) 5 on host 'smithi171' 2023-03-27T10:19:36.454 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:36.455+0000 7f79136b8700 1 -- 172.21.15.171:0/1011055164 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f78f809b3a0 msgr2=0x7f78f809d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:19:36.454 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:36.455+0000 7f79136b8700 1 --2- 172.21.15.171:0/1011055164 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f78f809b3a0 0x7f78f809d850 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f790c067de0 tx=0x7f78fc00b410 comp rx=0 tx=0).stop 2023-03-27T10:19:36.454 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:36.455+0000 7f79136b8700 1 -- 172.21.15.171:0/1011055164 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f790c101100 msgr2=0x7f790c196100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:19:36.454 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:36.455+0000 7f79136b8700 1 --2- 172.21.15.171:0/1011055164 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f790c101100 0x7f790c196100 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f790800d980 tx=0x7f790800dc90 comp rx=0 tx=0).stop 2023-03-27T10:19:36.455 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:36.455+0000 7f79136b8700 1 -- 172.21.15.171:0/1011055164 shutdown_connections 2023-03-27T10:19:36.455 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:36.455+0000 7f79136b8700 1 --2- 172.21.15.171:0/1011055164 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f78f809b3a0 0x7f78f809d850 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:36.455 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:36.455+0000 7f79136b8700 1 --2- 172.21.15.171:0/1011055164 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f790c101a90 0x7f790c19d180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:36.455 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:36.455+0000 7f79136b8700 1 --2- 172.21.15.171:0/1011055164 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f790c0fe080 0x7f790c195bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:36.455 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:36.455+0000 7f79136b8700 1 --2- 172.21.15.171:0/1011055164 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f790c101100 0x7f790c196100 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:36.455 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:36.455+0000 7f79136b8700 1 -- 172.21.15.171:0/1011055164 >> 172.21.15.171:0/1011055164 conn(0x7f790c0f9a30 msgr2=0x7f790c104530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:19:36.455 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:36.455+0000 7f79136b8700 1 -- 172.21.15.171:0/1011055164 shutdown_connections 2023-03-27T10:19:36.456 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:19:36.455+0000 7f79136b8700 1 -- 172.21.15.171:0/1011055164 wait complete. 2023-03-27T10:19:36.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:36 smithi171 ceph-mon[94808]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:36.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:36 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:36.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:36 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:36.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:36 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:36.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:36 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:19:36.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:36 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:36.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:36 smithi161 ceph-mon[94333]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:36.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:36 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:36.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:36 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:36.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:36 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:36.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:36 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:19:36.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:36 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:36.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:36 smithi159 ceph-mon[93039]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:36.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:36.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:36.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:36.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:19:36.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:36 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:36.989 DEBUG:teuthology.orchestra.run.smithi171:osd.5> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.5.service 2023-03-27T10:19:36.991 INFO:tasks.cephadm:Deploying osd.6 on smithi159 with /dev/vg_nvme/lv_2... 2023-03-27T10:19:36.992 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-03-27T10:19:37.033 INFO:journalctl@ceph.osd.5.smithi171.stdout:-- Logs begin at Mon 2023-03-27 10:03:15 UTC. -- 2023-03-27T10:19:37.659 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:19:37.690 INFO:journalctl@ceph.osd.5.smithi171.stdout:Mar 27 10:19:37 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-5[106084]: 2023-03-27T10:19:37.374+0000 7f0419d94580 -1 osd.5 0 log_to_monitors true 2023-03-27T10:19:37.690 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:37 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:37.690 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:37 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:37.691 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:37 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:19:37.691 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:37 smithi171 ceph-mon[94808]: from='osd.5 [v2:172.21.15.171:6808/569611891,v1:172.21.15.171:6809/569611891]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-03-27T10:19:37.691 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:37 smithi171 ceph-mon[94808]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-03-27T10:19:37.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:37 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:37.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:37 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:37.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:37 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:19:37.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:37 smithi161 ceph-mon[94333]: from='osd.5 [v2:172.21.15.171:6808/569611891,v1:172.21.15.171:6809/569611891]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-03-27T10:19:37.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:37 smithi161 ceph-mon[94333]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-03-27T10:19:37.849 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:37 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:37.849 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:37 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:37.849 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:37 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:19:37.849 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:37 smithi159 ceph-mon[93039]: from='osd.5 [v2:172.21.15.171:6808/569611891,v1:172.21.15.171:6809/569611891]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-03-27T10:19:37.849 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:37 smithi159 ceph-mon[93039]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-03-27T10:19:38.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:38 smithi171 ceph-mon[94808]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:38.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:38 smithi171 ceph-mon[94808]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-03-27T10:19:38.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:38 smithi171 ceph-mon[94808]: osdmap e36: 6 total, 5 up, 6 in 2023-03-27T10:19:38.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:38 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-03-27T10:19:38.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:38 smithi171 ceph-mon[94808]: from='osd.5 [v2:172.21.15.171:6808/569611891,v1:172.21.15.171:6809/569611891]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:19:38.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:38 smithi171 ceph-mon[94808]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:19:38.728 INFO:journalctl@ceph.osd.5.smithi171.stdout:Mar 27 10:19:38 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-5[106084]: 2023-03-27T10:19:38.463+0000 7f040f40f700 -1 osd.5 0 waiting for initial osdmap 2023-03-27T10:19:38.728 INFO:journalctl@ceph.osd.5.smithi171.stdout:Mar 27 10:19:38 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-5[106084]: 2023-03-27T10:19:38.472+0000 7f0409a01700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-03-27T10:19:38.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:38 smithi161 ceph-mon[94333]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:38.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:38 smithi161 ceph-mon[94333]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-03-27T10:19:38.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:38 smithi161 ceph-mon[94333]: osdmap e36: 6 total, 5 up, 6 in 2023-03-27T10:19:38.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:38 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-03-27T10:19:38.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:38 smithi161 ceph-mon[94333]: from='osd.5 [v2:172.21.15.171:6808/569611891,v1:172.21.15.171:6809/569611891]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:19:38.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:38 smithi161 ceph-mon[94333]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:19:38.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:38 smithi159 ceph-mon[93039]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:38.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:38 smithi159 ceph-mon[93039]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-03-27T10:19:38.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:38 smithi159 ceph-mon[93039]: osdmap e36: 6 total, 5 up, 6 in 2023-03-27T10:19:38.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:38 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-03-27T10:19:38.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:38 smithi159 ceph-mon[93039]: from='osd.5 [v2:172.21.15.171:6808/569611891,v1:172.21.15.171:6809/569611891]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:19:38.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:38 smithi159 ceph-mon[93039]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:19:39.085 INFO:teuthology.orchestra.run.smithi159.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-03-27T10:19:39.086 INFO:teuthology.orchestra.run.smithi159.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-03-27T10:19:39.086 INFO:teuthology.orchestra.run.smithi159.stderr: stderr: 10+0 records in 2023-03-27T10:19:39.086 INFO:teuthology.orchestra.run.smithi159.stderr:10+0 records out 2023-03-27T10:19:39.086 INFO:teuthology.orchestra.run.smithi159.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110623 s, 948 MB/s 2023-03-27T10:19:39.086 INFO:teuthology.orchestra.run.smithi159.stderr:--> Zapping successful for: 2023-03-27T10:19:39.517 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch daemon add osd smithi159:vg_nvme/lv_2 2023-03-27T10:19:39.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:39 smithi171 ceph-mon[94808]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2023-03-27T10:19:39.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:39 smithi171 ceph-mon[94808]: osdmap e37: 6 total, 5 up, 6 in 2023-03-27T10:19:39.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:39 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-03-27T10:19:39.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:39 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-03-27T10:19:39.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:39 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:39.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:39 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:39.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:39 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:19:39.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:39 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:19:39.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:39 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:39.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:39 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:39.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:39 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:19:39.728 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:39 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:39.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:39 smithi161 ceph-mon[94333]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2023-03-27T10:19:39.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:39 smithi161 ceph-mon[94333]: osdmap e37: 6 total, 5 up, 6 in 2023-03-27T10:19:39.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:39 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-03-27T10:19:39.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:39 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-03-27T10:19:39.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:39 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:39.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:39 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:39.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:39 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:19:39.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:39 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:19:39.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:39 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:39.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:39 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:39.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:39 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:19:39.836 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:39 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:39.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:39 smithi159 ceph-mon[93039]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2023-03-27T10:19:39.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:39 smithi159 ceph-mon[93039]: osdmap e37: 6 total, 5 up, 6 in 2023-03-27T10:19:39.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:39 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-03-27T10:19:39.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:39 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-03-27T10:19:39.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:39 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:39.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:39 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:39.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:39 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:19:39.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:39 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:19:39.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:39 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:39.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:39 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:39.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:39 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:19:39.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:39 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:40.164 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:19:40.675 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:40 smithi159 ceph-mon[93039]: purged_snaps scrub starts 2023-03-27T10:19:40.675 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:40 smithi159 ceph-mon[93039]: purged_snaps scrub ok 2023-03-27T10:19:40.675 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:40 smithi159 ceph-mon[93039]: pgmap v123: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:40.676 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:40 smithi159 ceph-mon[93039]: Detected new or changed devices on smithi171 2023-03-27T10:19:40.676 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:40 smithi159 ceph-mon[93039]: Adjusting osd_memory_target on smithi171 to 8584M 2023-03-27T10:19:40.676 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:40 smithi159 ceph-mon[93039]: osd.5 [v2:172.21.15.171:6808/569611891,v1:172.21.15.171:6809/569611891] boot 2023-03-27T10:19:40.676 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:40 smithi159 ceph-mon[93039]: osdmap e38: 6 total, 6 up, 6 in 2023-03-27T10:19:40.676 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:40 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-03-27T10:19:40.676 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:40 smithi159 ceph-mon[93039]: osdmap e39: 6 total, 6 up, 6 in 2023-03-27T10:19:40.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:40 smithi171 ceph-mon[94808]: purged_snaps scrub starts 2023-03-27T10:19:40.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:40 smithi171 ceph-mon[94808]: purged_snaps scrub ok 2023-03-27T10:19:40.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:40 smithi171 ceph-mon[94808]: pgmap v123: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:40.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:40 smithi171 ceph-mon[94808]: Detected new or changed devices on smithi171 2023-03-27T10:19:40.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:40 smithi171 ceph-mon[94808]: Adjusting osd_memory_target on smithi171 to 8584M 2023-03-27T10:19:40.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:40 smithi171 ceph-mon[94808]: osd.5 [v2:172.21.15.171:6808/569611891,v1:172.21.15.171:6809/569611891] boot 2023-03-27T10:19:40.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:40 smithi171 ceph-mon[94808]: osdmap e38: 6 total, 6 up, 6 in 2023-03-27T10:19:40.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:40 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-03-27T10:19:40.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:40 smithi171 ceph-mon[94808]: osdmap e39: 6 total, 6 up, 6 in 2023-03-27T10:19:40.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:40 smithi161 ceph-mon[94333]: purged_snaps scrub starts 2023-03-27T10:19:40.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:40 smithi161 ceph-mon[94333]: purged_snaps scrub ok 2023-03-27T10:19:40.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:40 smithi161 ceph-mon[94333]: pgmap v123: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-03-27T10:19:40.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:40 smithi161 ceph-mon[94333]: Detected new or changed devices on smithi171 2023-03-27T10:19:40.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:40 smithi161 ceph-mon[94333]: Adjusting osd_memory_target on smithi171 to 8584M 2023-03-27T10:19:40.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:40 smithi161 ceph-mon[94333]: osd.5 [v2:172.21.15.171:6808/569611891,v1:172.21.15.171:6809/569611891] boot 2023-03-27T10:19:40.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:40 smithi161 ceph-mon[94333]: osdmap e38: 6 total, 6 up, 6 in 2023-03-27T10:19:40.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:40 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-03-27T10:19:40.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:40 smithi161 ceph-mon[94333]: osdmap e39: 6 total, 6 up, 6 in 2023-03-27T10:19:41.359 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.359+0000 7f3a6e17a700 1 -- 172.21.15.159:0/488339630 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3a68105a70 msgr2=0x7f3a68107ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:19:41.359 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.359+0000 7f3a6e17a700 1 --2- 172.21.15.159:0/488339630 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3a68105a70 0x7f3a68107ed0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f3a58009990 tx=0x7f3a58009ca0 comp rx=0 tx=0).stop 2023-03-27T10:19:41.360 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.359+0000 7f3a6e17a700 1 -- 172.21.15.159:0/488339630 shutdown_connections 2023-03-27T10:19:41.360 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.359+0000 7f3a6e17a700 1 --2- 172.21.15.159:0/488339630 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3a68105a70 0x7f3a68107ed0 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:41.360 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.359+0000 7f3a6e17a700 1 --2- 172.21.15.159:0/488339630 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f3a68100830 0x7f3a68102c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:41.360 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.359+0000 7f3a6e17a700 1 --2- 172.21.15.159:0/488339630 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f3a68103150 0x7f3a68105530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:41.360 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.359+0000 7f3a6e17a700 1 -- 172.21.15.159:0/488339630 >> 172.21.15.159:0/488339630 conn(0x7f3a680fa2b0 msgr2=0x7f3a680fc720 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:19:41.361 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.359+0000 7f3a6e17a700 1 -- 172.21.15.159:0/488339630 shutdown_connections 2023-03-27T10:19:41.361 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.359+0000 7f3a6e17a700 1 -- 172.21.15.159:0/488339630 wait complete. 2023-03-27T10:19:41.362 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.360+0000 7f3a6e17a700 1 Processor -- start 2023-03-27T10:19:41.363 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.360+0000 7f3a6e17a700 1 -- start start 2023-03-27T10:19:41.363 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a6e17a700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f3a68100830 0x7f3a68195c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:19:41.363 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a6e17a700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3a68103150 0x7f3a68196190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:19:41.363 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a6e17a700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f3a68105a70 0x7f3a6819d1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:19:41.363 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a6e17a700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a6819d7a0 con 0x7f3a68103150 2023-03-27T10:19:41.363 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a6e17a700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a6819d8e0 con 0x7f3a68105a70 2023-03-27T10:19:41.363 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a6e17a700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a6819daa0 con 0x7f3a68100830 2023-03-27T10:19:41.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a66ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3a68103150 0x7f3a68196190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:19:41.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a66ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3a68103150 0x7f3a68196190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:44960/0 (socket says 172.21.15.159:44960) 2023-03-27T10:19:41.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a66ffd700 1 -- 172.21.15.159:0/4199495911 learned_addr learned my addr 172.21.15.159:0/4199495911 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:19:41.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a677fe700 1 --2- 172.21.15.159:0/4199495911 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f3a68100830 0x7f3a68195c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:19:41.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a67fff700 1 --2- 172.21.15.159:0/4199495911 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f3a68105a70 0x7f3a6819d1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:19:41.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a66ffd700 1 -- 172.21.15.159:0/4199495911 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f3a68100830 msgr2=0x7f3a68195c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:19:41.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a66ffd700 1 --2- 172.21.15.159:0/4199495911 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f3a68100830 0x7f3a68195c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:41.364 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a66ffd700 1 -- 172.21.15.159:0/4199495911 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f3a68105a70 msgr2=0x7f3a6819d1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:19:41.365 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a66ffd700 1 --2- 172.21.15.159:0/4199495911 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f3a68105a70 0x7f3a6819d1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:41.365 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a66ffd700 1 -- 172.21.15.159:0/4199495911 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3a580095f0 con 0x7f3a68103150 2023-03-27T10:19:41.365 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a66ffd700 1 --2- 172.21.15.159:0/4199495911 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3a68103150 0x7f3a68196190 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f3a5c00c9b0 tx=0x7f3a5c00ccc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:19:41.365 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a64ff9700 1 -- 172.21.15.159:0/4199495911 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a5c007b00 con 0x7f3a68103150 2023-03-27T10:19:41.365 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a6e17a700 1 -- 172.21.15.159:0/4199495911 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3a6810b770 con 0x7f3a68103150 2023-03-27T10:19:41.365 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a64ff9700 1 -- 172.21.15.159:0/4199495911 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3a5c007c60 con 0x7f3a68103150 2023-03-27T10:19:41.365 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.361+0000 7f3a64ff9700 1 -- 172.21.15.159:0/4199495911 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a5c018660 con 0x7f3a68103150 2023-03-27T10:19:41.366 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.362+0000 7f3a6e17a700 1 -- 172.21.15.159:0/4199495911 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3a6810b5e0 con 0x7f3a68103150 2023-03-27T10:19:41.366 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.363+0000 7f3a6e17a700 1 -- 172.21.15.159:0/4199495911 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3a680fbe20 con 0x7f3a68103150 2023-03-27T10:19:41.366 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.363+0000 7f3a64ff9700 1 -- 172.21.15.159:0/4199495911 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f3a5c00f450 con 0x7f3a68103150 2023-03-27T10:19:41.366 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.364+0000 7f3a64ff9700 1 --2- 172.21.15.159:0/4199495911 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f3a5009b3a0 0x7f3a5009d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:19:41.367 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.364+0000 7f3a64ff9700 1 -- 172.21.15.159:0/4199495911 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(39..39 src has 1..39) v4 ==== 4562+0+0 (secure 0 0 0) 0x7f3a5c0c41c0 con 0x7f3a68103150 2023-03-27T10:19:41.367 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.364+0000 7f3a677fe700 1 --2- 172.21.15.159:0/4199495911 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f3a5009b3a0 0x7f3a5009d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:19:41.367 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.364+0000 7f3a677fe700 1 --2- 172.21.15.159:0/4199495911 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f3a5009b3a0 0x7f3a5009d850 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f3a68197130 tx=0x7f3a6000d040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:19:41.367 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.368+0000 7f3a64ff9700 1 -- 172.21.15.159:0/4199495911 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f3a5c0909d0 con 0x7f3a68103150 2023-03-27T10:19:41.559 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:41.560+0000 7f3a6e17a700 1 -- 172.21.15.159:0/4199495911 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) v1 -- 0x7f3a680617e0 con 0x7f3a5009b3a0 2023-03-27T10:19:42.674 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:42 smithi159 ceph-mon[93039]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 33 MiB used, 536 GiB / 536 GiB avail 2023-03-27T10:19:42.675 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:42 smithi159 ceph-mon[93039]: osdmap e40: 6 total, 6 up, 6 in 2023-03-27T10:19:42.675 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:42 smithi159 ceph-mon[93039]: from='client.14364 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:19:42.675 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:42 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:19:42.675 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:42 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:19:42.675 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:42 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:42.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:42 smithi171 ceph-mon[94808]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 33 MiB used, 536 GiB / 536 GiB avail 2023-03-27T10:19:42.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:42 smithi171 ceph-mon[94808]: osdmap e40: 6 total, 6 up, 6 in 2023-03-27T10:19:42.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:42 smithi171 ceph-mon[94808]: from='client.14364 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:19:42.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:42 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:19:42.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:42 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:19:42.728 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:42 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:42.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:42 smithi161 ceph-mon[94333]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 33 MiB used, 536 GiB / 536 GiB avail 2023-03-27T10:19:42.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:42 smithi161 ceph-mon[94333]: osdmap e40: 6 total, 6 up, 6 in 2023-03-27T10:19:42.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:42 smithi161 ceph-mon[94333]: from='client.14364 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:19:42.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:42 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:19:42.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:42 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:19:42.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:42 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:43.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:43 smithi161 ceph-mon[94333]: osdmap e41: 6 total, 6 up, 6 in 2023-03-27T10:19:43.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:43 smithi159 ceph-mon[93039]: osdmap e41: 6 total, 6 up, 6 in 2023-03-27T10:19:43.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:43 smithi171 ceph-mon[94808]: osdmap e41: 6 total, 6 up, 6 in 2023-03-27T10:19:44.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:44 smithi161 ceph-mon[94333]: pgmap v129: 1 pgs: 1 peering; 577 KiB data, 33 MiB used, 536 GiB / 536 GiB avail 2023-03-27T10:19:44.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:44 smithi161 ceph-mon[94333]: osdmap e42: 6 total, 6 up, 6 in 2023-03-27T10:19:44.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:44 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3703548909' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dff7d5f4-655f-4baa-9084-03591a4b3bb9"}]: dispatch 2023-03-27T10:19:44.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:44 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3703548909' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dff7d5f4-655f-4baa-9084-03591a4b3bb9"}]': finished 2023-03-27T10:19:44.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:44 smithi161 ceph-mon[94333]: osdmap e43: 7 total, 6 up, 7 in 2023-03-27T10:19:44.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-03-27T10:19:44.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:44 smithi159 ceph-mon[93039]: pgmap v129: 1 pgs: 1 peering; 577 KiB data, 33 MiB used, 536 GiB / 536 GiB avail 2023-03-27T10:19:44.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:44 smithi159 ceph-mon[93039]: osdmap e42: 6 total, 6 up, 6 in 2023-03-27T10:19:44.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:44 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3703548909' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dff7d5f4-655f-4baa-9084-03591a4b3bb9"}]: dispatch 2023-03-27T10:19:44.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:44 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3703548909' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dff7d5f4-655f-4baa-9084-03591a4b3bb9"}]': finished 2023-03-27T10:19:44.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:44 smithi159 ceph-mon[93039]: osdmap e43: 7 total, 6 up, 7 in 2023-03-27T10:19:44.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-03-27T10:19:44.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:44 smithi171 ceph-mon[94808]: pgmap v129: 1 pgs: 1 peering; 577 KiB data, 33 MiB used, 536 GiB / 536 GiB avail 2023-03-27T10:19:44.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:44 smithi171 ceph-mon[94808]: osdmap e42: 6 total, 6 up, 6 in 2023-03-27T10:19:44.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:44 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3703548909' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dff7d5f4-655f-4baa-9084-03591a4b3bb9"}]: dispatch 2023-03-27T10:19:44.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:44 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3703548909' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dff7d5f4-655f-4baa-9084-03591a4b3bb9"}]': finished 2023-03-27T10:19:44.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:44 smithi171 ceph-mon[94808]: osdmap e43: 7 total, 6 up, 7 in 2023-03-27T10:19:44.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-03-27T10:19:45.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:45 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/2071652998' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:19:45.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:45 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/2071652998' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:19:45.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:45 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/2071652998' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:19:46.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:46 smithi161 ceph-mon[94333]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-03-27T10:19:46.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:46 smithi159 ceph-mon[93039]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-03-27T10:19:46.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:46 smithi171 ceph-mon[94808]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-03-27T10:19:48.738 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:48 smithi159 ceph-mon[93039]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-03-27T10:19:48.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:48 smithi161 ceph-mon[94333]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-03-27T10:19:48.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:48 smithi171 ceph-mon[94808]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-03-27T10:19:50.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:50 smithi161 ceph-mon[94333]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 89 KiB/s, 0 objects/s recovering 2023-03-27T10:19:50.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:50 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-03-27T10:19:50.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:50 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:50.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:50 smithi159 ceph-mon[93039]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 89 KiB/s, 0 objects/s recovering 2023-03-27T10:19:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:50 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-03-27T10:19:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:50 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:50.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:50 smithi171 ceph-mon[94808]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 89 KiB/s, 0 objects/s recovering 2023-03-27T10:19:50.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:50 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-03-27T10:19:50.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:50 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:19:51.755 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:51 smithi159 ceph-mon[93039]: Deploying daemon osd.6 on smithi159 2023-03-27T10:19:51.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:51 smithi161 ceph-mon[94333]: Deploying daemon osd.6 on smithi159 2023-03-27T10:19:51.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:51 smithi171 ceph-mon[94808]: Deploying daemon osd.6 on smithi159 2023-03-27T10:19:52.803 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:52 smithi159 ceph-mon[93039]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-03-27T10:19:52.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:52 smithi161 ceph-mon[94333]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-03-27T10:19:52.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:52 smithi171 ceph-mon[94808]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-03-27T10:19:54.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:54 smithi161 ceph-mon[94333]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-03-27T10:19:54.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:54 smithi159 ceph-mon[93039]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-03-27T10:19:54.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:54 smithi171 ceph-mon[94808]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-03-27T10:19:55.814 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:55.815 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:55.815 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:55 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:19:55.815 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:55 smithi159 ceph-mon[93039]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-03-27T10:19:55.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:55.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:55.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:55 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:19:55.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:55 smithi171 ceph-mon[94808]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-03-27T10:19:56.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:56.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:56.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:55 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:19:56.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:55 smithi161 ceph-mon[94333]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-03-27T10:19:57.273 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:57.273+0000 7f3a64ff9700 1 -- 172.21.15.159:0/4199495911 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f3a680617e0 con 0x7f3a5009b3a0 2023-03-27T10:19:57.273 INFO:teuthology.orchestra.run.smithi159.stdout:Created osd(s) 6 on host 'smithi159' 2023-03-27T10:19:57.274 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:57.275+0000 7f3a6e17a700 1 -- 172.21.15.159:0/4199495911 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f3a5009b3a0 msgr2=0x7f3a5009d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:19:57.275 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:57.275+0000 7f3a6e17a700 1 --2- 172.21.15.159:0/4199495911 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f3a5009b3a0 0x7f3a5009d850 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f3a68197130 tx=0x7f3a6000d040 comp rx=0 tx=0).stop 2023-03-27T10:19:57.275 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:57.275+0000 7f3a6e17a700 1 -- 172.21.15.159:0/4199495911 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3a68103150 msgr2=0x7f3a68196190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:19:57.275 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:57.275+0000 7f3a6e17a700 1 --2- 172.21.15.159:0/4199495911 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3a68103150 0x7f3a68196190 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f3a5c00c9b0 tx=0x7f3a5c00ccc0 comp rx=0 tx=0).stop 2023-03-27T10:19:57.275 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:57.275+0000 7f3a6e17a700 1 -- 172.21.15.159:0/4199495911 shutdown_connections 2023-03-27T10:19:57.275 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:57.275+0000 7f3a6e17a700 1 --2- 172.21.15.159:0/4199495911 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f3a5009b3a0 0x7f3a5009d850 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:57.275 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:57.275+0000 7f3a6e17a700 1 --2- 172.21.15.159:0/4199495911 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f3a68100830 0x7f3a68195c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:57.275 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:57.275+0000 7f3a6e17a700 1 --2- 172.21.15.159:0/4199495911 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f3a68105a70 0x7f3a6819d1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:57.276 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:57.275+0000 7f3a6e17a700 1 --2- 172.21.15.159:0/4199495911 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f3a68103150 0x7f3a68196190 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:19:57.276 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:57.275+0000 7f3a6e17a700 1 -- 172.21.15.159:0/4199495911 >> 172.21.15.159:0/4199495911 conn(0x7f3a680fa2b0 msgr2=0x7f3a68101470 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:19:57.276 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:57.275+0000 7f3a6e17a700 1 -- 172.21.15.159:0/4199495911 shutdown_connections 2023-03-27T10:19:57.276 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:19:57.275+0000 7f3a6e17a700 1 -- 172.21.15.159:0/4199495911 wait complete. 2023-03-27T10:19:57.657 DEBUG:teuthology.orchestra.run.smithi159:osd.6> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.6.service 2023-03-27T10:19:57.660 INFO:tasks.cephadm:Deploying osd.7 on smithi161 with /dev/vg_nvme/lv_2... 2023-03-27T10:19:57.660 DEBUG:teuthology.orchestra.run.smithi161:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-03-27T10:19:57.692 INFO:journalctl@ceph.osd.6.smithi159.stdout:-- Logs begin at Mon 2023-03-27 10:03:12 UTC. -- 2023-03-27T10:19:58.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:57 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:58.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:57 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:58.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:57 smithi171 ceph-mon[94808]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-03-27T10:19:58.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:57 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:58.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:57 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:58.324 INFO:teuthology.orchestra.run.smithi161.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.b/config 2023-03-27T10:19:58.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:57 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:58.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:57 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:58.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:57 smithi161 ceph-mon[94333]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-03-27T10:19:58.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:57 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:58.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:57 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:58.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:57 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:58.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:57 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:58.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:57 smithi159 ceph-mon[93039]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-03-27T10:19:58.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:57 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:58.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:57 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:19:58.751 INFO:journalctl@ceph.osd.6.smithi159.stdout:Mar 27 10:19:58 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-6[116815]: 2023-03-27T10:19:58.556+0000 7f8ebdf07580 -1 osd.6 0 log_to_monitors true 2023-03-27T10:19:59.113 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:58 smithi159 ceph-mon[93039]: from='osd.6 [v2:172.21.15.159:6818/1176978350,v1:172.21.15.159:6819/1176978350]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-03-27T10:19:59.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:58 smithi171 ceph-mon[94808]: from='osd.6 [v2:172.21.15.159:6818/1176978350,v1:172.21.15.159:6819/1176978350]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-03-27T10:19:59.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:58 smithi161 ceph-mon[94333]: from='osd.6 [v2:172.21.15.159:6818/1176978350,v1:172.21.15.159:6819/1176978350]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-03-27T10:20:00.005 INFO:teuthology.orchestra.run.smithi161.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-03-27T10:20:00.005 INFO:teuthology.orchestra.run.smithi161.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-03-27T10:20:00.006 INFO:teuthology.orchestra.run.smithi161.stderr: stderr: 10+0 records in 2023-03-27T10:20:00.006 INFO:teuthology.orchestra.run.smithi161.stderr:10+0 records out 2023-03-27T10:20:00.006 INFO:teuthology.orchestra.run.smithi161.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.011169 s, 939 MB/s 2023-03-27T10:20:00.006 INFO:teuthology.orchestra.run.smithi161.stderr:--> Zapping successful for: 2023-03-27T10:20:00.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:59 smithi161 ceph-mon[94333]: from='osd.6 [v2:172.21.15.159:6818/1176978350,v1:172.21.15.159:6819/1176978350]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-03-27T10:20:00.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:59 smithi161 ceph-mon[94333]: osdmap e44: 7 total, 6 up, 7 in 2023-03-27T10:20:00.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:59 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-03-27T10:20:00.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:59 smithi161 ceph-mon[94333]: from='osd.6 [v2:172.21.15.159:6818/1176978350,v1:172.21.15.159:6819/1176978350]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2023-03-27T10:20:00.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:19:59 smithi161 ceph-mon[94333]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-03-27T10:20:00.365 INFO:journalctl@ceph.osd.6.smithi159.stdout:Mar 27 10:19:59 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-6[116815]: 2023-03-27T10:19:59.987+0000 7f8eb3582700 -1 osd.6 0 waiting for initial osdmap 2023-03-27T10:20:00.365 INFO:journalctl@ceph.osd.6.smithi159.stdout:Mar 27 10:19:59 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-6[116815]: 2023-03-27T10:19:59.997+0000 7f8eac371700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-03-27T10:20:00.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:59 smithi159 ceph-mon[93039]: from='osd.6 [v2:172.21.15.159:6818/1176978350,v1:172.21.15.159:6819/1176978350]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-03-27T10:20:00.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:59 smithi159 ceph-mon[93039]: osdmap e44: 7 total, 6 up, 7 in 2023-03-27T10:20:00.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:59 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-03-27T10:20:00.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:59 smithi159 ceph-mon[93039]: from='osd.6 [v2:172.21.15.159:6818/1176978350,v1:172.21.15.159:6819/1176978350]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2023-03-27T10:20:00.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:19:59 smithi159 ceph-mon[93039]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-03-27T10:20:00.465 DEBUG:teuthology.orchestra.run.smithi161:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch daemon add osd smithi161:vg_nvme/lv_2 2023-03-27T10:20:00.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:59 smithi171 ceph-mon[94808]: from='osd.6 [v2:172.21.15.159:6818/1176978350,v1:172.21.15.159:6819/1176978350]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-03-27T10:20:00.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:59 smithi171 ceph-mon[94808]: osdmap e44: 7 total, 6 up, 7 in 2023-03-27T10:20:00.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:59 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-03-27T10:20:00.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:59 smithi171 ceph-mon[94808]: from='osd.6 [v2:172.21.15.159:6818/1176978350,v1:172.21.15.159:6819/1176978350]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2023-03-27T10:20:00.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:19:59 smithi171 ceph-mon[94808]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-03-27T10:20:01.103 INFO:teuthology.orchestra.run.smithi161.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.b/config 2023-03-27T10:20:01.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: from='osd.6 [v2:172.21.15.159:6818/1176978350,v1:172.21.15.159:6819/1176978350]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2023-03-27T10:20:01.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: osdmap e45: 7 total, 6 up, 7 in 2023-03-27T10:20:01.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-03-27T10:20:01.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: overall HEALTH_OK 2023-03-27T10:20:01.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: Detected new or changed devices on smithi159 2023-03-27T10:20:01.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:01.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:01.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:01.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:01.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:01.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: Adjusting osd_memory_target on smithi159 to 5722M 2023-03-27T10:20:01.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:01.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:01.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:01.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:01.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:20:01.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:01.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:01.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:01.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: from='osd.6 [v2:172.21.15.159:6818/1176978350,v1:172.21.15.159:6819/1176978350]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2023-03-27T10:20:01.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: osdmap e45: 7 total, 6 up, 7 in 2023-03-27T10:20:01.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-03-27T10:20:01.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: overall HEALTH_OK 2023-03-27T10:20:01.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: Detected new or changed devices on smithi159 2023-03-27T10:20:01.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:01.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:01.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:01.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:01.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:01.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: Adjusting osd_memory_target on smithi159 to 5722M 2023-03-27T10:20:01.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:01.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:01.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:01.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:01.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:20:01.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:01.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:01.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:01.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: from='osd.6 [v2:172.21.15.159:6818/1176978350,v1:172.21.15.159:6819/1176978350]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2023-03-27T10:20:01.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: osdmap e45: 7 total, 6 up, 7 in 2023-03-27T10:20:01.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-03-27T10:20:01.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: overall HEALTH_OK 2023-03-27T10:20:01.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: Detected new or changed devices on smithi159 2023-03-27T10:20:01.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:01.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:01.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:01.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:01.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:01.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: Adjusting osd_memory_target on smithi159 to 5722M 2023-03-27T10:20:01.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:01.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:01.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:01.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:01.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:20:01.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:01.479 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:01.479 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:02.321 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.321+0000 7f91cea9f700 1 -- 172.21.15.161:0/1071203791 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f91c80feca0 msgr2=0x7f91c80ff0b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:20:02.321 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.321+0000 7f91cea9f700 1 --2- 172.21.15.161:0/1071203791 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f91c80feca0 0x7f91c80ff0b0 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f91bc009990 tx=0x7f91bc009ca0 comp rx=0 tx=0).stop 2023-03-27T10:20:02.321 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.321+0000 7f91cea9f700 1 -- 172.21.15.161:0/1071203791 shutdown_connections 2023-03-27T10:20:02.321 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.321+0000 7f91cea9f700 1 --2- 172.21.15.161:0/1071203791 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f91c8100830 0x7f91c81070f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:02.321 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.321+0000 7f91cea9f700 1 --2- 172.21.15.161:0/1071203791 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f91c80feca0 0x7f91c80ff0b0 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:02.322 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.321+0000 7f91cea9f700 1 --2- 172.21.15.161:0/1071203791 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f91c80ffea0 0x7f91c81002f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:02.322 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.321+0000 7f91cea9f700 1 -- 172.21.15.161:0/1071203791 >> 172.21.15.161:0/1071203791 conn(0x7f91c80fa2d0 msgr2=0x7f91c80fc720 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:20:02.322 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.322+0000 7f91cea9f700 1 -- 172.21.15.161:0/1071203791 shutdown_connections 2023-03-27T10:20:02.322 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.322+0000 7f91cea9f700 1 -- 172.21.15.161:0/1071203791 wait complete. 2023-03-27T10:20:02.324 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.322+0000 7f91cea9f700 1 Processor -- start 2023-03-27T10:20:02.324 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.322+0000 7f91cea9f700 1 -- start start 2023-03-27T10:20:02.324 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.322+0000 7f91cea9f700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f91c80feca0 0x7f91c8195b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:20:02.324 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.322+0000 7f91cea9f700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f91c80ffea0 0x7f91c8196060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:20:02.325 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.322+0000 7f91cea9f700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f91c8100830 0x7f91c819d0e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:20:02.325 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.322+0000 7f91cea9f700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f91c819d6b0 con 0x7f91c8100830 2023-03-27T10:20:02.325 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.322+0000 7f91cea9f700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f91c819d7f0 con 0x7f91c80ffea0 2023-03-27T10:20:02.325 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.322+0000 7f91cea9f700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f91c819d980 con 0x7f91c80feca0 2023-03-27T10:20:02.325 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.322+0000 7f91c7fff700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f91c80ffea0 0x7f91c8196060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:20:02.326 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.322+0000 7f91c7fff700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f91c80ffea0 0x7f91c8196060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.161:3300/0 says I am v2:172.21.15.161:44682/0 (socket says 172.21.15.161:44682) 2023-03-27T10:20:02.326 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.322+0000 7f91c7fff700 1 -- 172.21.15.161:0/3525325396 learned_addr learned my addr 172.21.15.161:0/3525325396 (peer_addr_for_me v2:172.21.15.161:0/0) 2023-03-27T10:20:02.326 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.323+0000 7f91cd03c700 1 --2- 172.21.15.161:0/3525325396 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f91c8100830 0x7f91c819d0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:20:02.326 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.323+0000 7f91cc83b700 1 --2- 172.21.15.161:0/3525325396 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f91c80feca0 0x7f91c8195b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:20:02.326 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.323+0000 7f91c7fff700 1 -- 172.21.15.161:0/3525325396 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f91c80feca0 msgr2=0x7f91c8195b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:20:02.326 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.323+0000 7f91c7fff700 1 --2- 172.21.15.161:0/3525325396 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f91c80feca0 0x7f91c8195b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:02.326 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.323+0000 7f91c7fff700 1 -- 172.21.15.161:0/3525325396 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f91c8100830 msgr2=0x7f91c819d0e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:20:02.327 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.323+0000 7f91c7fff700 1 --2- 172.21.15.161:0/3525325396 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f91c8100830 0x7f91c819d0e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:02.327 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.323+0000 7f91c7fff700 1 -- 172.21.15.161:0/3525325396 --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f91bc0095f0 con 0x7f91c80ffea0 2023-03-27T10:20:02.327 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.323+0000 7f91c7fff700 1 --2- 172.21.15.161:0/3525325396 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f91c80ffea0 0x7f91c8196060 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f91b400c920 tx=0x7f91b400cc30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:20:02.327 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.323+0000 7f91c5ffb700 1 -- 172.21.15.161:0/3525325396 <== mon.2 v2:172.21.15.161:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91b4004a30 con 0x7f91c80ffea0 2023-03-27T10:20:02.327 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.323+0000 7f91cea9f700 1 -- 172.21.15.161:0/3525325396 --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f91c810a5f0 con 0x7f91c80ffea0 2023-03-27T10:20:02.327 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.323+0000 7f91c5ffb700 1 -- 172.21.15.161:0/3525325396 <== mon.2 v2:172.21.15.161:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f91b40054e0 con 0x7f91c80ffea0 2023-03-27T10:20:02.327 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.323+0000 7f91c5ffb700 1 -- 172.21.15.161:0/3525325396 <== mon.2 v2:172.21.15.161:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91b4007c10 con 0x7f91c80ffea0 2023-03-27T10:20:02.328 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.323+0000 7f91cea9f700 1 -- 172.21.15.161:0/3525325396 --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f91c810a460 con 0x7f91c80ffea0 2023-03-27T10:20:02.328 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.324+0000 7f91cea9f700 1 -- 172.21.15.161:0/3525325396 --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f91c819e080 con 0x7f91c80ffea0 2023-03-27T10:20:02.328 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.325+0000 7f91c5ffb700 1 -- 172.21.15.161:0/3525325396 <== mon.2 v2:172.21.15.161:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f91b4007d70 con 0x7f91c80ffea0 2023-03-27T10:20:02.329 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.326+0000 7f91c5ffb700 1 --2- 172.21.15.161:0/3525325396 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f91b809b3a0 0x7f91b809d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:20:02.329 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.326+0000 7f91c5ffb700 1 -- 172.21.15.161:0/3525325396 <== mon.2 v2:172.21.15.161:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 4965+0+0 (secure 0 0 0) 0x7f91b40c41f0 con 0x7f91c80ffea0 2023-03-27T10:20:02.329 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.326+0000 7f91cc83b700 1 --2- 172.21.15.161:0/3525325396 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f91b809b3a0 0x7f91b809d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:20:02.329 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.326+0000 7f91cc83b700 1 --2- 172.21.15.161:0/3525325396 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f91b809b3a0 0x7f91b809d850 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f91bc00b5c0 tx=0x7f91bc005fd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:20:02.330 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.330+0000 7f91c5ffb700 1 -- 172.21.15.161:0/3525325396 <== mon.2 v2:172.21.15.161:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f91b4090870 con 0x7f91c80ffea0 2023-03-27T10:20:02.333 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:01 smithi161 ceph-mon[94333]: purged_snaps scrub starts 2023-03-27T10:20:02.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:01 smithi161 ceph-mon[94333]: purged_snaps scrub ok 2023-03-27T10:20:02.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:01 smithi161 ceph-mon[94333]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-03-27T10:20:02.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:01 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-03-27T10:20:02.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:01 smithi161 ceph-mon[94333]: osd.6 [v2:172.21.15.159:6818/1176978350,v1:172.21.15.159:6819/1176978350] boot 2023-03-27T10:20:02.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:01 smithi161 ceph-mon[94333]: osdmap e46: 7 total, 7 up, 7 in 2023-03-27T10:20:02.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:01 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-03-27T10:20:02.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:01 smithi159 ceph-mon[93039]: purged_snaps scrub starts 2023-03-27T10:20:02.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:01 smithi159 ceph-mon[93039]: purged_snaps scrub ok 2023-03-27T10:20:02.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:01 smithi159 ceph-mon[93039]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-03-27T10:20:02.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:01 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-03-27T10:20:02.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:01 smithi159 ceph-mon[93039]: osd.6 [v2:172.21.15.159:6818/1176978350,v1:172.21.15.159:6819/1176978350] boot 2023-03-27T10:20:02.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:01 smithi159 ceph-mon[93039]: osdmap e46: 7 total, 7 up, 7 in 2023-03-27T10:20:02.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:01 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-03-27T10:20:02.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:01 smithi171 ceph-mon[94808]: purged_snaps scrub starts 2023-03-27T10:20:02.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:01 smithi171 ceph-mon[94808]: purged_snaps scrub ok 2023-03-27T10:20:02.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:01 smithi171 ceph-mon[94808]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-03-27T10:20:02.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:01 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-03-27T10:20:02.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:01 smithi171 ceph-mon[94808]: osd.6 [v2:172.21.15.159:6818/1176978350,v1:172.21.15.159:6819/1176978350] boot 2023-03-27T10:20:02.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:01 smithi171 ceph-mon[94808]: osdmap e46: 7 total, 7 up, 7 in 2023-03-27T10:20:02.478 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:01 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-03-27T10:20:02.521 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:02.522+0000 7f91cea9f700 1 -- 172.21.15.161:0/3525325396 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi161:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) v1 -- 0x7f91c819e630 con 0x7f91b809b3a0 2023-03-27T10:20:03.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:03 smithi161 ceph-mon[94333]: osdmap e47: 7 total, 7 up, 7 in 2023-03-27T10:20:03.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:03 smithi161 ceph-mon[94333]: from='client.24272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi161:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:20:03.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:03 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:20:03.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:03 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:20:03.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:03 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:03.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:03 smithi159 ceph-mon[93039]: osdmap e47: 7 total, 7 up, 7 in 2023-03-27T10:20:03.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:03 smithi159 ceph-mon[93039]: from='client.24272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi161:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:20:03.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:03 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:20:03.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:03 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:20:03.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:03 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:03.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:03 smithi171 ceph-mon[94808]: osdmap e47: 7 total, 7 up, 7 in 2023-03-27T10:20:03.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:03 smithi171 ceph-mon[94808]: from='client.24272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi161:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:20:03.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:20:03.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:20:03.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:04.297 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:04 smithi161 ceph-mon[94333]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:04.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:04 smithi159 ceph-mon[93039]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:04.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:04 smithi171 ceph-mon[94808]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:06.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:06 smithi161 ceph-mon[94333]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:06.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:06 smithi161 ceph-mon[94333]: from='client.? 172.21.15.161:0/3516840606' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30f026d9-6963-4468-95e8-78c7ed2e89f5"}]: dispatch 2023-03-27T10:20:06.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:06 smithi161 ceph-mon[94333]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30f026d9-6963-4468-95e8-78c7ed2e89f5"}]: dispatch 2023-03-27T10:20:06.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:06 smithi161 ceph-mon[94333]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "30f026d9-6963-4468-95e8-78c7ed2e89f5"}]': finished 2023-03-27T10:20:06.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:06 smithi161 ceph-mon[94333]: osdmap e48: 8 total, 7 up, 8 in 2023-03-27T10:20:06.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-03-27T10:20:06.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:06 smithi161 ceph-mon[94333]: from='client.? 172.21.15.161:0/1886899959' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:20:06.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:06 smithi159 ceph-mon[93039]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:06.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:06 smithi159 ceph-mon[93039]: from='client.? 172.21.15.161:0/3516840606' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30f026d9-6963-4468-95e8-78c7ed2e89f5"}]: dispatch 2023-03-27T10:20:06.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:06 smithi159 ceph-mon[93039]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30f026d9-6963-4468-95e8-78c7ed2e89f5"}]: dispatch 2023-03-27T10:20:06.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:06 smithi159 ceph-mon[93039]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "30f026d9-6963-4468-95e8-78c7ed2e89f5"}]': finished 2023-03-27T10:20:06.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:06 smithi159 ceph-mon[93039]: osdmap e48: 8 total, 7 up, 8 in 2023-03-27T10:20:06.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-03-27T10:20:06.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:06 smithi159 ceph-mon[93039]: from='client.? 172.21.15.161:0/1886899959' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:20:06.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:06 smithi171 ceph-mon[94808]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:06.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:06 smithi171 ceph-mon[94808]: from='client.? 172.21.15.161:0/3516840606' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30f026d9-6963-4468-95e8-78c7ed2e89f5"}]: dispatch 2023-03-27T10:20:06.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:06 smithi171 ceph-mon[94808]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30f026d9-6963-4468-95e8-78c7ed2e89f5"}]: dispatch 2023-03-27T10:20:06.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:06 smithi171 ceph-mon[94808]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "30f026d9-6963-4468-95e8-78c7ed2e89f5"}]': finished 2023-03-27T10:20:06.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:06 smithi171 ceph-mon[94808]: osdmap e48: 8 total, 7 up, 8 in 2023-03-27T10:20:06.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-03-27T10:20:06.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:06 smithi171 ceph-mon[94808]: from='client.? 172.21.15.161:0/1886899959' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:20:08.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:08 smithi161 ceph-mon[94333]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:08.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:08 smithi159 ceph-mon[93039]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:08.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:08 smithi171 ceph-mon[94808]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:10.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:10 smithi161 ceph-mon[94333]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:10.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:10 smithi159 ceph-mon[93039]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:10.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:10 smithi171 ceph-mon[94808]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:12.696 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:12 smithi161 ceph-mon[94333]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:12.696 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:12 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-03-27T10:20:12.697 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:12 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:12.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:12 smithi171 ceph-mon[94808]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:12.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:12 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-03-27T10:20:12.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:12 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:12.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:12 smithi159 ceph-mon[93039]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:12.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:12 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-03-27T10:20:12.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:12 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:13.581 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:13 smithi161 ceph-mon[94333]: Deploying daemon osd.7 on smithi161 2023-03-27T10:20:13.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:13 smithi171 ceph-mon[94808]: Deploying daemon osd.7 on smithi161 2023-03-27T10:20:13.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:13 smithi159 ceph-mon[93039]: Deploying daemon osd.7 on smithi161 2023-03-27T10:20:14.579 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:14 smithi161 ceph-mon[94333]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:14.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:14 smithi171 ceph-mon[94808]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:14.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:14 smithi159 ceph-mon[93039]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:16.690 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:16 smithi171 ceph-mon[94808]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:16.691 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:16 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:16.691 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:16 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:20:16.691 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:16 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:16.821 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:16 smithi161 ceph-mon[94333]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:16.821 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:16 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:16.821 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:16 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:20:16.821 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:16 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:16.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:16 smithi159 ceph-mon[93039]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:16.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:16 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:16.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:16 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:20:16.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:16 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:18.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:18 smithi171 ceph-mon[94808]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:18.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:18 smithi161 ceph-mon[94333]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:18.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:18 smithi159 ceph-mon[93039]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:19.327 INFO:teuthology.orchestra.run.smithi161.stdout:Created osd(s) 7 on host 'smithi161' 2023-03-27T10:20:19.328 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:19.326+0000 7f91c5ffb700 1 -- 172.21.15.161:0/3525325396 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f91c819e630 con 0x7f91b809b3a0 2023-03-27T10:20:19.329 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:19.329+0000 7f91cea9f700 1 -- 172.21.15.161:0/3525325396 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f91b809b3a0 msgr2=0x7f91b809d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:20:19.329 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:19.329+0000 7f91cea9f700 1 --2- 172.21.15.161:0/3525325396 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f91b809b3a0 0x7f91b809d850 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f91bc00b5c0 tx=0x7f91bc005fd0 comp rx=0 tx=0).stop 2023-03-27T10:20:19.329 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:19.329+0000 7f91cea9f700 1 -- 172.21.15.161:0/3525325396 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f91c80ffea0 msgr2=0x7f91c8196060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:20:19.329 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:19.329+0000 7f91cea9f700 1 --2- 172.21.15.161:0/3525325396 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f91c80ffea0 0x7f91c8196060 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f91b400c920 tx=0x7f91b400cc30 comp rx=0 tx=0).stop 2023-03-27T10:20:19.329 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:19.329+0000 7f91cea9f700 1 -- 172.21.15.161:0/3525325396 shutdown_connections 2023-03-27T10:20:19.329 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:19.329+0000 7f91cea9f700 1 --2- 172.21.15.161:0/3525325396 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f91b809b3a0 0x7f91b809d850 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:19.330 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:19.329+0000 7f91cea9f700 1 --2- 172.21.15.161:0/3525325396 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f91c80feca0 0x7f91c8195b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:19.330 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:19.329+0000 7f91cea9f700 1 --2- 172.21.15.161:0/3525325396 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f91c8100830 0x7f91c819d0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:19.330 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:19.329+0000 7f91cea9f700 1 --2- 172.21.15.161:0/3525325396 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f91c80ffea0 0x7f91c8196060 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:19.330 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:19.329+0000 7f91cea9f700 1 -- 172.21.15.161:0/3525325396 >> 172.21.15.161:0/3525325396 conn(0x7f91c80fa2d0 msgr2=0x7f91c8068970 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:20:19.330 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:19.329+0000 7f91cea9f700 1 -- 172.21.15.161:0/3525325396 shutdown_connections 2023-03-27T10:20:19.330 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:20:19.329+0000 7f91cea9f700 1 -- 172.21.15.161:0/3525325396 wait complete. 2023-03-27T10:20:19.800 DEBUG:teuthology.orchestra.run.smithi161:osd.7> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.7.service 2023-03-27T10:20:19.803 INFO:tasks.cephadm:Deploying osd.8 on smithi171 with /dev/vg_nvme/lv_2... 2023-03-27T10:20:19.803 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-03-27T10:20:19.833 INFO:journalctl@ceph.osd.7.smithi161.stdout:-- Logs begin at Mon 2023-03-27 10:02:59 UTC. -- 2023-03-27T10:20:20.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:19 smithi171 ceph-mon[94808]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:20.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:19 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:20.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:19 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:20.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:19 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:20.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:19 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:20.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:19 smithi161 ceph-mon[94333]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:20.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:19 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:20.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:19 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:20.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:19 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:20.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:19 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:20.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:19 smithi159 ceph-mon[93039]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:20.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:19 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:20.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:19 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:20.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:19 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:20.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:19 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:20.456 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.c/config 2023-03-27T10:20:20.649 INFO:journalctl@ceph.osd.7.smithi161.stdout:Mar 27 10:20:20 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-7[110246]: 2023-03-27T10:20:20.352+0000 7fceec822580 -1 osd.7 0 log_to_monitors true 2023-03-27T10:20:21.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:20 smithi171 ceph-mon[94808]: from='osd.7 [v2:172.21.15.161:6816/2098174471,v1:172.21.15.161:6817/2098174471]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-03-27T10:20:21.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:20 smithi171 ceph-mon[94808]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-03-27T10:20:21.271 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:20 smithi161 ceph-mon[94333]: from='osd.7 [v2:172.21.15.161:6816/2098174471,v1:172.21.15.161:6817/2098174471]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-03-27T10:20:21.272 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:20 smithi161 ceph-mon[94333]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-03-27T10:20:21.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:20 smithi159 ceph-mon[93039]: from='osd.7 [v2:172.21.15.161:6816/2098174471,v1:172.21.15.161:6817/2098174471]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-03-27T10:20:21.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:20 smithi159 ceph-mon[93039]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-03-27T10:20:22.012 INFO:teuthology.orchestra.run.smithi171.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-03-27T10:20:22.012 INFO:teuthology.orchestra.run.smithi171.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-03-27T10:20:22.012 INFO:teuthology.orchestra.run.smithi171.stderr: stderr: 10+0 records in 2023-03-27T10:20:22.013 INFO:teuthology.orchestra.run.smithi171.stderr:10+0 records out 2023-03-27T10:20:22.013 INFO:teuthology.orchestra.run.smithi171.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0126031 s, 832 MB/s 2023-03-27T10:20:22.013 INFO:teuthology.orchestra.run.smithi171.stderr:--> Zapping successful for: 2023-03-27T10:20:22.158 INFO:journalctl@ceph.osd.7.smithi161.stdout:Mar 27 10:20:22 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-7[110246]: 2023-03-27T10:20:22.022+0000 7fcee1e9d700 -1 osd.7 0 waiting for initial osdmap 2023-03-27T10:20:22.158 INFO:journalctl@ceph.osd.7.smithi161.stdout:Mar 27 10:20:22 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-7[110246]: 2023-03-27T10:20:22.032+0000 7fcedcc90700 -1 osd.7 50 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-03-27T10:20:22.159 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:22 smithi161 ceph-mon[94333]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:22.159 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:22 smithi161 ceph-mon[94333]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-03-27T10:20:22.159 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:22 smithi161 ceph-mon[94333]: osdmap e49: 8 total, 7 up, 8 in 2023-03-27T10:20:22.159 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:22 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-03-27T10:20:22.159 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:22 smithi161 ceph-mon[94333]: from='osd.7 [v2:172.21.15.161:6816/2098174471,v1:172.21.15.161:6817/2098174471]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:20:22.159 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:22 smithi161 ceph-mon[94333]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:20:22.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:22 smithi159 ceph-mon[93039]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:22.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:22 smithi159 ceph-mon[93039]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-03-27T10:20:22.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:22 smithi159 ceph-mon[93039]: osdmap e49: 8 total, 7 up, 8 in 2023-03-27T10:20:22.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:22 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-03-27T10:20:22.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:22 smithi159 ceph-mon[93039]: from='osd.7 [v2:172.21.15.161:6816/2098174471,v1:172.21.15.161:6817/2098174471]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:20:22.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:22 smithi159 ceph-mon[93039]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:20:22.468 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:22 smithi171 ceph-mon[94808]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:22.468 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:22 smithi171 ceph-mon[94808]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-03-27T10:20:22.468 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:22 smithi171 ceph-mon[94808]: osdmap e49: 8 total, 7 up, 8 in 2023-03-27T10:20:22.468 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:22 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-03-27T10:20:22.468 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:22 smithi171 ceph-mon[94808]: from='osd.7 [v2:172.21.15.161:6816/2098174471,v1:172.21.15.161:6817/2098174471]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:20:22.469 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:22 smithi171 ceph-mon[94808]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:20:22.499 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch daemon add osd smithi171:vg_nvme/lv_2 2023-03-27T10:20:23.138 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.c/config 2023-03-27T10:20:23.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]': finished 2023-03-27T10:20:23.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: osdmap e50: 8 total, 7 up, 8 in 2023-03-27T10:20:23.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-03-27T10:20:23.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-03-27T10:20:23.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: Detected new or changed devices on smithi161 2023-03-27T10:20:23.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:23.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:23.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:23.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:23.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:23.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: Adjusting osd_memory_target on smithi161 to 5722M 2023-03-27T10:20:23.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:23.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:23.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:23.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:23.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:20:23.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:23.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:23.336 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:23.337 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: osd.7 [v2:172.21.15.161:6816/2098174471,v1:172.21.15.161:6817/2098174471] boot 2023-03-27T10:20:23.337 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: osdmap e51: 8 total, 8 up, 8 in 2023-03-27T10:20:23.337 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:23 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-03-27T10:20:23.347 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]': finished 2023-03-27T10:20:23.347 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: osdmap e50: 8 total, 7 up, 8 in 2023-03-27T10:20:23.347 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-03-27T10:20:23.348 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-03-27T10:20:23.348 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: Detected new or changed devices on smithi161 2023-03-27T10:20:23.348 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:23.348 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:23.348 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:23.348 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:23.348 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:23.349 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: Adjusting osd_memory_target on smithi161 to 5722M 2023-03-27T10:20:23.349 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:23.349 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:23.349 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:23.349 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:23.349 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:20:23.349 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:23.350 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:23.350 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:23.350 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: osd.7 [v2:172.21.15.161:6816/2098174471,v1:172.21.15.161:6817/2098174471] boot 2023-03-27T10:20:23.350 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: osdmap e51: 8 total, 8 up, 8 in 2023-03-27T10:20:23.350 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:23 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-03-27T10:20:23.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]': finished 2023-03-27T10:20:23.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: osdmap e50: 8 total, 7 up, 8 in 2023-03-27T10:20:23.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-03-27T10:20:23.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-03-27T10:20:23.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: Detected new or changed devices on smithi161 2023-03-27T10:20:23.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:23.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:23.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:23.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:23.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:23.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: Adjusting osd_memory_target on smithi161 to 5722M 2023-03-27T10:20:23.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:23.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:23.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:23.366 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:23.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:20:23.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:23.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:23.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:23.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: osd.7 [v2:172.21.15.161:6816/2098174471,v1:172.21.15.161:6817/2098174471] boot 2023-03-27T10:20:23.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: osdmap e51: 8 total, 8 up, 8 in 2023-03-27T10:20:23.367 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:23 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-03-27T10:20:24.364 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.365+0000 7f84a9b15700 1 -- 172.21.15.171:0/2067580993 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f84a40ffe50 msgr2=0x7f84a4100260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:20:24.365 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.365+0000 7f84a9b15700 1 --2- 172.21.15.171:0/2067580993 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f84a40ffe50 0x7f84a4100260 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f8498009990 tx=0x7f8498009ca0 comp rx=0 tx=0).stop 2023-03-27T10:20:24.365 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.365+0000 7f84a9b15700 1 -- 172.21.15.171:0/2067580993 shutdown_connections 2023-03-27T10:20:24.365 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.365+0000 7f84a9b15700 1 --2- 172.21.15.171:0/2067580993 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f84a40ffe50 0x7f84a4100260 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:24.365 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.365+0000 7f84a9b15700 1 --2- 172.21.15.171:0/2067580993 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f84a4101960 0x7f84a4108290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:24.365 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.365+0000 7f84a9b15700 1 --2- 172.21.15.171:0/2067580993 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f84a4100fb0 0x7f84a4101420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:24.365 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.365+0000 7f84a9b15700 1 -- 172.21.15.171:0/2067580993 >> 172.21.15.171:0/2067580993 conn(0x7f84a40fb480 msgr2=0x7f84a40fd8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:20:24.366 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.365+0000 7f84a9b15700 1 -- 172.21.15.171:0/2067580993 shutdown_connections 2023-03-27T10:20:24.366 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.365+0000 7f84a9b15700 1 -- 172.21.15.171:0/2067580993 wait complete. 2023-03-27T10:20:24.366 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.365+0000 7f84a9b15700 1 Processor -- start 2023-03-27T10:20:24.366 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a9b15700 1 -- start start 2023-03-27T10:20:24.366 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a9b15700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f84a40ffe50 0x7f84a4074570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:20:24.366 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a9b15700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f84a4100fb0 0x7f84a4072bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:20:24.366 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a9b15700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f84a4101960 0x7f84a4073100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:20:24.367 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a9b15700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84a4073880 con 0x7f84a40ffe50 2023-03-27T10:20:24.367 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a9b15700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84a40739f0 con 0x7f84a4101960 2023-03-27T10:20:24.367 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a9b15700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84a419fc50 con 0x7f84a4100fb0 2023-03-27T10:20:24.367 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a2ffd700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f84a4100fb0 0x7f84a4072bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:20:24.367 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a2ffd700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f84a4100fb0 0x7f84a4072bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.171:3300/0 says I am v2:172.21.15.171:55330/0 (socket says 172.21.15.171:55330) 2023-03-27T10:20:24.367 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a2ffd700 1 -- 172.21.15.171:0/3089900865 learned_addr learned my addr 172.21.15.171:0/3089900865 (peer_addr_for_me v2:172.21.15.171:0/0) 2023-03-27T10:20:24.367 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a3fff700 1 --2- 172.21.15.171:0/3089900865 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f84a4101960 0x7f84a4073100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:20:24.369 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a37fe700 1 --2- 172.21.15.171:0/3089900865 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f84a40ffe50 0x7f84a4074570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:20:24.369 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a2ffd700 1 -- 172.21.15.171:0/3089900865 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f84a4101960 msgr2=0x7f84a4073100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:20:24.369 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a2ffd700 1 --2- 172.21.15.171:0/3089900865 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f84a4101960 0x7f84a4073100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:24.369 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a2ffd700 1 -- 172.21.15.171:0/3089900865 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f84a40ffe50 msgr2=0x7f84a4074570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:20:24.369 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a2ffd700 1 --2- 172.21.15.171:0/3089900865 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f84a40ffe50 0x7f84a4074570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:24.369 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a2ffd700 1 -- 172.21.15.171:0/3089900865 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f84980095f0 con 0x7f84a4100fb0 2023-03-27T10:20:24.370 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a2ffd700 1 --2- 172.21.15.171:0/3089900865 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f84a4100fb0 0x7f84a4072bc0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f849400b530 tx=0x7f849400b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:20:24.370 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a0ff9700 1 -- 172.21.15.171:0/3089900865 <== mon.1 v2:172.21.15.171:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8494011830 con 0x7f84a4100fb0 2023-03-27T10:20:24.370 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a9b15700 1 -- 172.21.15.171:0/3089900865 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f84a410b8f0 con 0x7f84a4100fb0 2023-03-27T10:20:24.370 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a0ff9700 1 -- 172.21.15.171:0/3089900865 <== mon.1 v2:172.21.15.171:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8494011e50 con 0x7f84a4100fb0 2023-03-27T10:20:24.370 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.366+0000 7f84a0ff9700 1 -- 172.21.15.171:0/3089900865 <== mon.1 v2:172.21.15.171:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f849400f2e0 con 0x7f84a4100fb0 2023-03-27T10:20:24.370 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.367+0000 7f84a9b15700 1 -- 172.21.15.171:0/3089900865 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f84a410b760 con 0x7f84a4100fb0 2023-03-27T10:20:24.370 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.368+0000 7f84a9b15700 1 -- 172.21.15.171:0/3089900865 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f84a41073e0 con 0x7f84a4100fb0 2023-03-27T10:20:24.370 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.369+0000 7f84a0ff9700 1 -- 172.21.15.171:0/3089900865 <== mon.1 v2:172.21.15.171:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f8494021070 con 0x7f84a4100fb0 2023-03-27T10:20:24.371 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.369+0000 7f84a0ff9700 1 --2- 172.21.15.171:0/3089900865 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f848c09b3a0 0x7f848c09d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:20:24.371 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.369+0000 7f84a0ff9700 1 -- 172.21.15.171:0/3089900865 <== mon.1 v2:172.21.15.171:3300/0 5 ==== osd_map(52..52 src has 1..52) v4 ==== 5426+0+0 (secure 0 0 0) 0x7f84940c3db0 con 0x7f84a4100fb0 2023-03-27T10:20:24.371 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.369+0000 7f84a37fe700 1 --2- 172.21.15.171:0/3089900865 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f848c09b3a0 0x7f848c09d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:20:24.371 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.370+0000 7f84a37fe700 1 --2- 172.21.15.171:0/3089900865 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f848c09b3a0 0x7f848c09d850 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f8498000c00 tx=0x7f8498005fb0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:20:24.372 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.373+0000 7f84a0ff9700 1 -- 172.21.15.171:0/3089900865 <== mon.1 v2:172.21.15.171:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f8494090260 con 0x7f84a4100fb0 2023-03-27T10:20:24.475 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:24 smithi171 ceph-mon[94808]: purged_snaps scrub starts 2023-03-27T10:20:24.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:24 smithi171 ceph-mon[94808]: purged_snaps scrub ok 2023-03-27T10:20:24.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:24 smithi171 ceph-mon[94808]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:24.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:24 smithi171 ceph-mon[94808]: osdmap e52: 8 total, 8 up, 8 in 2023-03-27T10:20:24.561 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:24.562+0000 7f84a9b15700 1 -- 172.21.15.171:0/3089900865 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) v1 -- 0x7f84a40618f0 con 0x7f848c09b3a0 2023-03-27T10:20:24.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:24 smithi159 ceph-mon[93039]: purged_snaps scrub starts 2023-03-27T10:20:24.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:24 smithi159 ceph-mon[93039]: purged_snaps scrub ok 2023-03-27T10:20:24.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:24 smithi159 ceph-mon[93039]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:24.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:24 smithi159 ceph-mon[93039]: osdmap e52: 8 total, 8 up, 8 in 2023-03-27T10:20:24.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:24 smithi161 ceph-mon[94333]: purged_snaps scrub starts 2023-03-27T10:20:24.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:24 smithi161 ceph-mon[94333]: purged_snaps scrub ok 2023-03-27T10:20:24.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:24 smithi161 ceph-mon[94333]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-03-27T10:20:24.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:24 smithi161 ceph-mon[94333]: osdmap e52: 8 total, 8 up, 8 in 2023-03-27T10:20:25.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:25 smithi159 ceph-mon[93039]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:20:25.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:25 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:20:25.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:25 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:20:25.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:25 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:25.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:25 smithi159 ceph-mon[93039]: osdmap e53: 8 total, 8 up, 8 in 2023-03-27T10:20:25.715 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:25 smithi171 ceph-mon[94808]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:20:25.716 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:25 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:20:25.716 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:25 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:20:25.716 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:25 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:25.716 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:25 smithi171 ceph-mon[94808]: osdmap e53: 8 total, 8 up, 8 in 2023-03-27T10:20:25.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:25 smithi161 ceph-mon[94333]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:20:25.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:25 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:20:25.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:25 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:20:25.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:25 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:25.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:25 smithi161 ceph-mon[94333]: osdmap e53: 8 total, 8 up, 8 in 2023-03-27T10:20:26.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:26 smithi171 ceph-mon[94808]: pgmap v161: 1 pgs: 1 remapped+peering; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:26.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:26 smithi171 ceph-mon[94808]: osdmap e54: 8 total, 8 up, 8 in 2023-03-27T10:20:26.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:26 smithi161 ceph-mon[94333]: pgmap v161: 1 pgs: 1 remapped+peering; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:26.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:26 smithi161 ceph-mon[94333]: osdmap e54: 8 total, 8 up, 8 in 2023-03-27T10:20:26.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:26 smithi159 ceph-mon[93039]: pgmap v161: 1 pgs: 1 remapped+peering; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:26.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:26 smithi159 ceph-mon[93039]: osdmap e54: 8 total, 8 up, 8 in 2023-03-27T10:20:27.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:27 smithi171 ceph-mon[94808]: osdmap e55: 8 total, 8 up, 8 in 2023-03-27T10:20:27.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:27 smithi171 ceph-mon[94808]: from='client.? 172.21.15.171:0/3446262269' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f69fbcc-32ff-4d4b-b562-ee0a12745cc2"}]: dispatch 2023-03-27T10:20:27.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:27 smithi171 ceph-mon[94808]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f69fbcc-32ff-4d4b-b562-ee0a12745cc2"}]: dispatch 2023-03-27T10:20:27.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:27 smithi171 ceph-mon[94808]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9f69fbcc-32ff-4d4b-b562-ee0a12745cc2"}]': finished 2023-03-27T10:20:27.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:27 smithi171 ceph-mon[94808]: osdmap e56: 9 total, 8 up, 9 in 2023-03-27T10:20:27.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:27 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-03-27T10:20:27.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:27 smithi161 ceph-mon[94333]: osdmap e55: 8 total, 8 up, 8 in 2023-03-27T10:20:27.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:27 smithi161 ceph-mon[94333]: from='client.? 172.21.15.171:0/3446262269' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f69fbcc-32ff-4d4b-b562-ee0a12745cc2"}]: dispatch 2023-03-27T10:20:27.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:27 smithi161 ceph-mon[94333]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f69fbcc-32ff-4d4b-b562-ee0a12745cc2"}]: dispatch 2023-03-27T10:20:27.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:27 smithi161 ceph-mon[94333]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9f69fbcc-32ff-4d4b-b562-ee0a12745cc2"}]': finished 2023-03-27T10:20:27.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:27 smithi161 ceph-mon[94333]: osdmap e56: 9 total, 8 up, 9 in 2023-03-27T10:20:27.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:27 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-03-27T10:20:27.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:27 smithi159 ceph-mon[93039]: osdmap e55: 8 total, 8 up, 8 in 2023-03-27T10:20:27.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:27 smithi159 ceph-mon[93039]: from='client.? 172.21.15.171:0/3446262269' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f69fbcc-32ff-4d4b-b562-ee0a12745cc2"}]: dispatch 2023-03-27T10:20:27.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:27 smithi159 ceph-mon[93039]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f69fbcc-32ff-4d4b-b562-ee0a12745cc2"}]: dispatch 2023-03-27T10:20:27.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:27 smithi159 ceph-mon[93039]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9f69fbcc-32ff-4d4b-b562-ee0a12745cc2"}]': finished 2023-03-27T10:20:27.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:27 smithi159 ceph-mon[93039]: osdmap e56: 9 total, 8 up, 9 in 2023-03-27T10:20:27.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:27 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-03-27T10:20:28.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:28 smithi171 ceph-mon[94808]: pgmap v165: 1 pgs: 1 remapped+peering; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:28.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:28 smithi171 ceph-mon[94808]: from='client.? 172.21.15.171:0/2222977136' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:20:28.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:28 smithi161 ceph-mon[94333]: pgmap v165: 1 pgs: 1 remapped+peering; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:28.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:28 smithi161 ceph-mon[94333]: from='client.? 172.21.15.171:0/2222977136' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:20:28.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:28 smithi159 ceph-mon[93039]: pgmap v165: 1 pgs: 1 remapped+peering; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:28.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:28 smithi159 ceph-mon[93039]: from='client.? 172.21.15.171:0/2222977136' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:20:30.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:30 smithi171 ceph-mon[94808]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:30.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:30 smithi161 ceph-mon[94333]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:30.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:30 smithi159 ceph-mon[93039]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:32.718 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:32 smithi171 ceph-mon[94808]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:32.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:32 smithi161 ceph-mon[94333]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:32.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:32 smithi159 ceph-mon[93039]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:34.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:34 smithi171 ceph-mon[94808]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:34.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:34 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2023-03-27T10:20:34.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:34 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:34.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:34 smithi161 ceph-mon[94333]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:34.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:34 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2023-03-27T10:20:34.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:34 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:34.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:34 smithi159 ceph-mon[93039]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:34.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:34 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2023-03-27T10:20:34.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:34 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:35.706 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:35 smithi171 ceph-mon[94808]: Deploying daemon osd.8 on smithi171 2023-03-27T10:20:35.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:35 smithi161 ceph-mon[94333]: Deploying daemon osd.8 on smithi171 2023-03-27T10:20:35.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:35 smithi159 ceph-mon[93039]: Deploying daemon osd.8 on smithi171 2023-03-27T10:20:36.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:36 smithi171 ceph-mon[94808]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:36.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:36 smithi161 ceph-mon[94333]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:36.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:36 smithi159 ceph-mon[93039]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:38.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:38 smithi171 ceph-mon[94808]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:38.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:38 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:38.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:38 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:38.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:38 smithi161 ceph-mon[94333]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:38.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:38 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:38.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:38 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:38.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:38 smithi159 ceph-mon[93039]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:38.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:38 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:38.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:38 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:39.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:39 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:20:39.833 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:39 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:20:39.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:39 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:20:40.473 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:40 smithi171 ceph-mon[94808]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:40.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:40 smithi161 ceph-mon[94333]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:40.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:40 smithi159 ceph-mon[93039]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:41.388 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:41.389+0000 7f84a0ff9700 1 -- 172.21.15.171:0/3089900865 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f84a40618f0 con 0x7f848c09b3a0 2023-03-27T10:20:41.388 INFO:teuthology.orchestra.run.smithi171.stdout:Created osd(s) 8 on host 'smithi171' 2023-03-27T10:20:41.390 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:41.391+0000 7f84a9b15700 1 -- 172.21.15.171:0/3089900865 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f848c09b3a0 msgr2=0x7f848c09d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:20:41.390 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:41.391+0000 7f84a9b15700 1 --2- 172.21.15.171:0/3089900865 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f848c09b3a0 0x7f848c09d850 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f8498000c00 tx=0x7f8498005fb0 comp rx=0 tx=0).stop 2023-03-27T10:20:41.390 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:41.391+0000 7f84a9b15700 1 -- 172.21.15.171:0/3089900865 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f84a4100fb0 msgr2=0x7f84a4072bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:20:41.390 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:41.391+0000 7f84a9b15700 1 --2- 172.21.15.171:0/3089900865 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f84a4100fb0 0x7f84a4072bc0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f849400b530 tx=0x7f849400b840 comp rx=0 tx=0).stop 2023-03-27T10:20:41.391 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:41.391+0000 7f84a9b15700 1 -- 172.21.15.171:0/3089900865 shutdown_connections 2023-03-27T10:20:41.391 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:41.391+0000 7f84a9b15700 1 --2- 172.21.15.171:0/3089900865 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f848c09b3a0 0x7f848c09d850 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:41.391 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:41.391+0000 7f84a9b15700 1 --2- 172.21.15.171:0/3089900865 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f84a4101960 0x7f84a4073100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:41.391 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:41.391+0000 7f84a9b15700 1 --2- 172.21.15.171:0/3089900865 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f84a40ffe50 0x7f84a4074570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:41.391 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:41.391+0000 7f84a9b15700 1 --2- 172.21.15.171:0/3089900865 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f84a4100fb0 0x7f84a4072bc0 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:41.391 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:41.391+0000 7f84a9b15700 1 -- 172.21.15.171:0/3089900865 >> 172.21.15.171:0/3089900865 conn(0x7f84a40fb480 msgr2=0x7f84a4104520 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:20:41.391 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:41.391+0000 7f84a9b15700 1 -- 172.21.15.171:0/3089900865 shutdown_connections 2023-03-27T10:20:41.391 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:20:41.392+0000 7f84a9b15700 1 -- 172.21.15.171:0/3089900865 wait complete. 2023-03-27T10:20:41.837 DEBUG:teuthology.orchestra.run.smithi171:osd.8> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.8.service 2023-03-27T10:20:41.839 INFO:tasks.cephadm:Deploying osd.9 on smithi159 with /dev/vg_nvme/lv_1... 2023-03-27T10:20:41.840 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-03-27T10:20:41.870 INFO:journalctl@ceph.osd.8.smithi171.stdout:-- Logs begin at Mon 2023-03-27 10:03:15 UTC. -- 2023-03-27T10:20:42.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:41 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:42.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:41 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:42.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:41 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:42.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:41 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:42.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:41 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:42.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:41 smithi161 ceph-mon[94333]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:42.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:41 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:42.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:41 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:20:42.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:41 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:42.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:41 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:42.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:41 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:42.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:41 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:42.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:41 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:42.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:41 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:42.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:41 smithi159 ceph-mon[93039]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:42.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:41 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:42.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:41 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:20:42.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:41 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:42.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:41 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:42.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:41 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:42.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:41 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:42.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:41 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:42.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:41 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:42.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:41 smithi171 ceph-mon[94808]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:42.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:41 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:42.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:41 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:20:42.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:41 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:42.495 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:20:42.596 INFO:journalctl@ceph.osd.8.smithi171.stdout:Mar 27 10:20:42 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-8[111586]: 2023-03-27T10:20:42.353+0000 7f6094d2b580 -1 osd.8 0 log_to_monitors true 2023-03-27T10:20:42.874 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:42 smithi171 ceph-mon[94808]: from='osd.8 [v2:172.21.15.171:6816/1033586906,v1:172.21.15.171:6817/1033586906]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2023-03-27T10:20:42.874 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:42 smithi171 ceph-mon[94808]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2023-03-27T10:20:42.982 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:42 smithi159 ceph-mon[93039]: from='osd.8 [v2:172.21.15.171:6816/1033586906,v1:172.21.15.171:6817/1033586906]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2023-03-27T10:20:42.982 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:42 smithi159 ceph-mon[93039]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2023-03-27T10:20:43.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:42 smithi161 ceph-mon[94333]: from='osd.8 [v2:172.21.15.171:6816/1033586906,v1:172.21.15.171:6817/1033586906]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2023-03-27T10:20:43.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:42 smithi161 ceph-mon[94333]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2023-03-27T10:20:43.976 INFO:journalctl@ceph.osd.8.smithi171.stdout:Mar 27 10:20:43 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-8[111586]: 2023-03-27T10:20:43.830+0000 7f608a3a6700 -1 osd.8 0 waiting for initial osdmap 2023-03-27T10:20:43.977 INFO:journalctl@ceph.osd.8.smithi171.stdout:Mar 27 10:20:43 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-8[111586]: 2023-03-27T10:20:43.841+0000 7f6084998700 -1 osd.8 58 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-03-27T10:20:43.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:43 smithi171 ceph-mon[94808]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2023-03-27T10:20:43.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:43 smithi171 ceph-mon[94808]: osdmap e57: 9 total, 8 up, 9 in 2023-03-27T10:20:43.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:43 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-03-27T10:20:43.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:43 smithi171 ceph-mon[94808]: from='osd.8 [v2:172.21.15.171:6816/1033586906,v1:172.21.15.171:6817/1033586906]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:20:43.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:43 smithi171 ceph-mon[94808]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:20:43.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:43 smithi171 ceph-mon[94808]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:44.000 INFO:teuthology.orchestra.run.smithi159.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-03-27T10:20:44.000 INFO:teuthology.orchestra.run.smithi159.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-03-27T10:20:44.000 INFO:teuthology.orchestra.run.smithi159.stderr: stderr: 10+0 records in 2023-03-27T10:20:44.000 INFO:teuthology.orchestra.run.smithi159.stderr:10+0 records out 2023-03-27T10:20:44.000 INFO:teuthology.orchestra.run.smithi159.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0109004 s, 962 MB/s 2023-03-27T10:20:44.000 INFO:teuthology.orchestra.run.smithi159.stderr:--> Zapping successful for: 2023-03-27T10:20:44.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:43 smithi161 ceph-mon[94333]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2023-03-27T10:20:44.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:43 smithi161 ceph-mon[94333]: osdmap e57: 9 total, 8 up, 9 in 2023-03-27T10:20:44.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:43 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-03-27T10:20:44.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:43 smithi161 ceph-mon[94333]: from='osd.8 [v2:172.21.15.171:6816/1033586906,v1:172.21.15.171:6817/1033586906]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:20:44.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:43 smithi161 ceph-mon[94333]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:20:44.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:43 smithi161 ceph-mon[94333]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:44.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:43 smithi159 ceph-mon[93039]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2023-03-27T10:20:44.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:43 smithi159 ceph-mon[93039]: osdmap e57: 9 total, 8 up, 9 in 2023-03-27T10:20:44.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:43 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-03-27T10:20:44.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:43 smithi159 ceph-mon[93039]: from='osd.8 [v2:172.21.15.171:6816/1033586906,v1:172.21.15.171:6817/1033586906]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:20:44.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:43 smithi159 ceph-mon[93039]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:20:44.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:43 smithi159 ceph-mon[93039]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-03-27T10:20:44.349 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch daemon add osd smithi159:vg_nvme/lv_1 2023-03-27T10:20:44.993 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:20:45.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2023-03-27T10:20:45.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: osdmap e58: 9 total, 8 up, 9 in 2023-03-27T10:20:45.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-03-27T10:20:45.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-03-27T10:20:45.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: Detected new or changed devices on smithi171 2023-03-27T10:20:45.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:45.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:45.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:45.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:45.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:45.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: Adjusting osd_memory_target on smithi171 to 5722M 2023-03-27T10:20:45.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:45.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:45.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:45.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:45.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: osd.8 [v2:172.21.15.171:6816/1033586906,v1:172.21.15.171:6817/1033586906] boot 2023-03-27T10:20:45.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: osdmap e59: 9 total, 9 up, 9 in 2023-03-27T10:20:45.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-03-27T10:20:45.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2023-03-27T10:20:45.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: osdmap e58: 9 total, 8 up, 9 in 2023-03-27T10:20:45.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-03-27T10:20:45.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-03-27T10:20:45.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: Detected new or changed devices on smithi171 2023-03-27T10:20:45.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:45.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:45.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:45.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:45.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:45.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: Adjusting osd_memory_target on smithi171 to 5722M 2023-03-27T10:20:45.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:45.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:45.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:45.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:45.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: osd.8 [v2:172.21.15.171:6816/1033586906,v1:172.21.15.171:6817/1033586906] boot 2023-03-27T10:20:45.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: osdmap e59: 9 total, 9 up, 9 in 2023-03-27T10:20:45.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-03-27T10:20:45.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2023-03-27T10:20:45.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: osdmap e58: 9 total, 8 up, 9 in 2023-03-27T10:20:45.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-03-27T10:20:45.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-03-27T10:20:45.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: Detected new or changed devices on smithi171 2023-03-27T10:20:45.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:45.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:45.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:45.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:45.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:20:45.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: Adjusting osd_memory_target on smithi171 to 5722M 2023-03-27T10:20:45.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:45.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:45.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:20:45.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:20:45.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: osd.8 [v2:172.21.15.171:6816/1033586906,v1:172.21.15.171:6817/1033586906] boot 2023-03-27T10:20:45.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: osdmap e59: 9 total, 9 up, 9 in 2023-03-27T10:20:45.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-03-27T10:20:46.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:45 smithi161 ceph-mon[94333]: purged_snaps scrub starts 2023-03-27T10:20:46.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:45 smithi161 ceph-mon[94333]: purged_snaps scrub ok 2023-03-27T10:20:46.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:45 smithi161 ceph-mon[94333]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 52 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:20:46.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:45 smithi161 ceph-mon[94333]: osdmap e60: 9 total, 9 up, 9 in 2023-03-27T10:20:46.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:45 smithi159 ceph-mon[93039]: purged_snaps scrub starts 2023-03-27T10:20:46.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:45 smithi159 ceph-mon[93039]: purged_snaps scrub ok 2023-03-27T10:20:46.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:45 smithi159 ceph-mon[93039]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 52 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:20:46.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:45 smithi159 ceph-mon[93039]: osdmap e60: 9 total, 9 up, 9 in 2023-03-27T10:20:46.220 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.220+0000 7fd9523c3700 1 -- 172.21.15.159:0/3926327818 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd94c0ffea0 msgr2=0x7fd94c1002f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:20:46.220 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.220+0000 7fd9523c3700 1 --2- 172.21.15.159:0/3926327818 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd94c0ffea0 0x7fd94c1002f0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7fd940009990 tx=0x7fd940009ca0 comp rx=0 tx=0).stop 2023-03-27T10:20:46.220 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.220+0000 7fd9523c3700 1 -- 172.21.15.159:0/3926327818 shutdown_connections 2023-03-27T10:20:46.220 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.220+0000 7fd9523c3700 1 --2- 172.21.15.159:0/3926327818 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd94c100830 0x7fd94c1070f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:46.221 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.220+0000 7fd9523c3700 1 --2- 172.21.15.159:0/3926327818 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd94c0feca0 0x7fd94c0ff0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:46.221 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.220+0000 7fd9523c3700 1 --2- 172.21.15.159:0/3926327818 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd94c0ffea0 0x7fd94c1002f0 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:46.221 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.220+0000 7fd9523c3700 1 -- 172.21.15.159:0/3926327818 >> 172.21.15.159:0/3926327818 conn(0x7fd94c0fa2b0 msgr2=0x7fd94c0fc720 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:20:46.221 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.220+0000 7fd9523c3700 1 -- 172.21.15.159:0/3926327818 shutdown_connections 2023-03-27T10:20:46.221 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.220+0000 7fd9523c3700 1 -- 172.21.15.159:0/3926327818 wait complete. 2023-03-27T10:20:46.223 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.221+0000 7fd9523c3700 1 Processor -- start 2023-03-27T10:20:46.224 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.221+0000 7fd9523c3700 1 -- start start 2023-03-27T10:20:46.224 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.221+0000 7fd9523c3700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd94c0feca0 0x7fd94c102c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:20:46.224 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.221+0000 7fd9523c3700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd94c0ffea0 0x7fd94c101280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:20:46.224 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.221+0000 7fd9523c3700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd94c100830 0x7fd94c1017c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:20:46.224 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.221+0000 7fd9523c3700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd94c101f40 con 0x7fd94c100830 2023-03-27T10:20:46.224 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.221+0000 7fd9523c3700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd94c1020b0 con 0x7fd94c0ffea0 2023-03-27T10:20:46.224 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.221+0000 7fd9523c3700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd94c19fbc0 con 0x7fd94c0feca0 2023-03-27T10:20:46.225 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.221+0000 7fd950960700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd94c100830 0x7fd94c1017c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:20:46.225 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.221+0000 7fd950960700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd94c100830 0x7fd94c1017c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45100/0 (socket says 172.21.15.159:45100) 2023-03-27T10:20:46.225 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.221+0000 7fd950960700 1 -- 172.21.15.159:0/3069999361 learned_addr learned my addr 172.21.15.159:0/3069999361 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:20:46.225 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.222+0000 7fd94b7fe700 1 --2- 172.21.15.159:0/3069999361 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd94c0ffea0 0x7fd94c101280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:20:46.225 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.222+0000 7fd94bfff700 1 --2- 172.21.15.159:0/3069999361 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd94c0feca0 0x7fd94c102c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:20:46.225 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.222+0000 7fd950960700 1 -- 172.21.15.159:0/3069999361 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd94c0feca0 msgr2=0x7fd94c102c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:20:46.225 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.222+0000 7fd950960700 1 --2- 172.21.15.159:0/3069999361 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd94c0feca0 0x7fd94c102c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:46.226 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.222+0000 7fd950960700 1 -- 172.21.15.159:0/3069999361 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd94c0ffea0 msgr2=0x7fd94c101280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:20:46.226 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.222+0000 7fd950960700 1 --2- 172.21.15.159:0/3069999361 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd94c0ffea0 0x7fd94c101280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:20:46.226 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.222+0000 7fd950960700 1 -- 172.21.15.159:0/3069999361 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd9400095f0 con 0x7fd94c100830 2023-03-27T10:20:46.226 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.222+0000 7fd94bfff700 1 --2- 172.21.15.159:0/3069999361 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd94c0feca0 0x7fd94c102c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-03-27T10:20:46.226 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.222+0000 7fd950960700 1 --2- 172.21.15.159:0/3069999361 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd94c100830 0x7fd94c1017c0 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7fd94400b970 tx=0x7fd94400bd30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:20:46.226 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.222+0000 7fd9497fa700 1 -- 172.21.15.159:0/3069999361 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd94400c9b0 con 0x7fd94c100830 2023-03-27T10:20:46.226 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.222+0000 7fd9497fa700 1 -- 172.21.15.159:0/3069999361 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd944014440 con 0x7fd94c100830 2023-03-27T10:20:46.227 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.222+0000 7fd9523c3700 1 -- 172.21.15.159:0/3069999361 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd94c10a6c0 con 0x7fd94c100830 2023-03-27T10:20:46.227 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.222+0000 7fd9497fa700 1 -- 172.21.15.159:0/3069999361 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd9440075d0 con 0x7fd94c100830 2023-03-27T10:20:46.227 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.222+0000 7fd9523c3700 1 -- 172.21.15.159:0/3069999361 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd94c10a530 con 0x7fd94c100830 2023-03-27T10:20:46.227 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.223+0000 7fd9523c3700 1 -- 172.21.15.159:0/3069999361 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd94c1a0220 con 0x7fd94c100830 2023-03-27T10:20:46.227 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.224+0000 7fd9497fa700 1 -- 172.21.15.159:0/3069999361 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7fd94401b030 con 0x7fd94c100830 2023-03-27T10:20:46.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:45 smithi171 ceph-mon[94808]: purged_snaps scrub starts 2023-03-27T10:20:46.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:45 smithi171 ceph-mon[94808]: purged_snaps scrub ok 2023-03-27T10:20:46.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:45 smithi171 ceph-mon[94808]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 52 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:20:46.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:45 smithi171 ceph-mon[94808]: osdmap e60: 9 total, 9 up, 9 in 2023-03-27T10:20:46.229 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.225+0000 7fd9497fa700 1 --2- 172.21.15.159:0/3069999361 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd93409b3a0 0x7fd93409d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:20:46.229 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.225+0000 7fd9497fa700 1 -- 172.21.15.159:0/3069999361 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(60..60 src has 1..60) v4 ==== 5858+0+0 (secure 0 0 0) 0x7fd944006e40 con 0x7fd94c100830 2023-03-27T10:20:46.229 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.225+0000 7fd94bfff700 1 --2- 172.21.15.159:0/3069999361 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd93409b3a0 0x7fd93409d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:20:46.229 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.225+0000 7fd94bfff700 1 --2- 172.21.15.159:0/3069999361 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd93409b3a0 0x7fd93409d850 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fd93c005fd0 tx=0x7fd93c005dc0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:20:46.230 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.229+0000 7fd9497fa700 1 -- 172.21.15.159:0/3069999361 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fd94404d960 con 0x7fd94c100830 2023-03-27T10:20:46.421 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:20:46.421+0000 7fd9523c3700 1 -- 172.21.15.159:0/3069999361 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) v1 -- 0x7fd94c1a07d0 con 0x7fd93409b3a0 2023-03-27T10:20:47.598 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:47 smithi159 ceph-mon[93039]: osdmap e61: 9 total, 9 up, 9 in 2023-03-27T10:20:47.598 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:47 smithi159 ceph-mon[93039]: from='client.14424 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:20:47.599 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:47 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:20:47.599 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:47 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:20:47.599 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:47 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:47.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:47 smithi171 ceph-mon[94808]: osdmap e61: 9 total, 9 up, 9 in 2023-03-27T10:20:47.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:47 smithi171 ceph-mon[94808]: from='client.14424 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:20:47.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:47 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:20:47.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:47 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:20:47.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:47 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:47.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:47 smithi161 ceph-mon[94333]: osdmap e61: 9 total, 9 up, 9 in 2023-03-27T10:20:47.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:47 smithi161 ceph-mon[94333]: from='client.14424 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:20:47.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:47 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:20:47.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:47 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:20:47.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:47 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:48.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:48 smithi171 ceph-mon[94808]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 52 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:20:48.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:48 smithi171 ceph-mon[94808]: osdmap e62: 9 total, 9 up, 9 in 2023-03-27T10:20:48.833 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:48 smithi161 ceph-mon[94333]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 52 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:20:48.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:48 smithi161 ceph-mon[94333]: osdmap e62: 9 total, 9 up, 9 in 2023-03-27T10:20:48.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:48 smithi159 ceph-mon[93039]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 52 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:20:48.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:48 smithi159 ceph-mon[93039]: osdmap e62: 9 total, 9 up, 9 in 2023-03-27T10:20:49.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:49 smithi171 ceph-mon[94808]: osdmap e63: 9 total, 9 up, 9 in 2023-03-27T10:20:49.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:49 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3333128727' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d7fb9eed-901b-4943-9529-6edcba2385e7"}]: dispatch 2023-03-27T10:20:49.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:49 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3333128727' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d7fb9eed-901b-4943-9529-6edcba2385e7"}]': finished 2023-03-27T10:20:49.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:49 smithi171 ceph-mon[94808]: osdmap e64: 10 total, 9 up, 10 in 2023-03-27T10:20:49.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:20:49.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:49 smithi161 ceph-mon[94333]: osdmap e63: 9 total, 9 up, 9 in 2023-03-27T10:20:49.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:49 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3333128727' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d7fb9eed-901b-4943-9529-6edcba2385e7"}]: dispatch 2023-03-27T10:20:49.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:49 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3333128727' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d7fb9eed-901b-4943-9529-6edcba2385e7"}]': finished 2023-03-27T10:20:49.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:49 smithi161 ceph-mon[94333]: osdmap e64: 10 total, 9 up, 10 in 2023-03-27T10:20:49.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:20:49.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:49 smithi159 ceph-mon[93039]: osdmap e63: 9 total, 9 up, 9 in 2023-03-27T10:20:49.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:49 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3333128727' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d7fb9eed-901b-4943-9529-6edcba2385e7"}]: dispatch 2023-03-27T10:20:49.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:49 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3333128727' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d7fb9eed-901b-4943-9529-6edcba2385e7"}]': finished 2023-03-27T10:20:49.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:49 smithi159 ceph-mon[93039]: osdmap e64: 10 total, 9 up, 10 in 2023-03-27T10:20:49.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:20:50.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:50 smithi171 ceph-mon[94808]: pgmap v185: 1 pgs: 1 peering; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail; 161 KiB/s, 0 objects/s recovering 2023-03-27T10:20:50.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:50 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/1530837743' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:20:50.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:50 smithi161 ceph-mon[94333]: pgmap v185: 1 pgs: 1 peering; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail; 161 KiB/s, 0 objects/s recovering 2023-03-27T10:20:50.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:50 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/1530837743' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:20:50.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:50 smithi159 ceph-mon[93039]: pgmap v185: 1 pgs: 1 peering; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail; 161 KiB/s, 0 objects/s recovering 2023-03-27T10:20:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:50 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/1530837743' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:20:52.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:52 smithi171 ceph-mon[94808]: pgmap v186: 1 pgs: 1 peering; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail; 126 KiB/s, 0 objects/s recovering 2023-03-27T10:20:52.833 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:52 smithi161 ceph-mon[94333]: pgmap v186: 1 pgs: 1 peering; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail; 126 KiB/s, 0 objects/s recovering 2023-03-27T10:20:52.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:52 smithi159 ceph-mon[93039]: pgmap v186: 1 pgs: 1 peering; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail; 126 KiB/s, 0 objects/s recovering 2023-03-27T10:20:54.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:54 smithi159 ceph-mon[93039]: pgmap v187: 1 pgs: 1 peering; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-03-27T10:20:54.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:54 smithi171 ceph-mon[94808]: pgmap v187: 1 pgs: 1 peering; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-03-27T10:20:54.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:54 smithi161 ceph-mon[94333]: pgmap v187: 1 pgs: 1 peering; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-03-27T10:20:56.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:56 smithi171 ceph-mon[94808]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail; 76 KiB/s, 0 objects/s recovering 2023-03-27T10:20:56.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:56 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2023-03-27T10:20:56.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:56 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:56.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:56 smithi171 ceph-mon[94808]: Deploying daemon osd.9 on smithi159 2023-03-27T10:20:56.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:56 smithi161 ceph-mon[94333]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail; 76 KiB/s, 0 objects/s recovering 2023-03-27T10:20:56.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:56 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2023-03-27T10:20:56.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:56 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:56.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:56 smithi161 ceph-mon[94333]: Deploying daemon osd.9 on smithi159 2023-03-27T10:20:56.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:56 smithi159 ceph-mon[93039]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail; 76 KiB/s, 0 objects/s recovering 2023-03-27T10:20:56.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:56 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2023-03-27T10:20:56.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:56 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:20:56.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:56 smithi159 ceph-mon[93039]: Deploying daemon osd.9 on smithi159 2023-03-27T10:20:58.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:20:58 smithi159 ceph-mon[93039]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:20:58.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:20:58 smithi171 ceph-mon[94808]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:20:58.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:20:58 smithi161 ceph-mon[94333]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:21:00.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:00 smithi159 ceph-mon[93039]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:21:00.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:00.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:00.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:00.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:00 smithi171 ceph-mon[94808]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:21:00.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:00.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:00.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:00.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:00 smithi161 ceph-mon[94333]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:21:00.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:00.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:00.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:02.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:02 smithi171 ceph-mon[94808]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:21:02.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:02 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:02.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:02 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:02.766 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:02.766+0000 7fd9497fa700 1 -- 172.21.15.159:0/3069999361 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7fd94c1a07d0 con 0x7fd93409b3a0 2023-03-27T10:21:02.766 INFO:teuthology.orchestra.run.smithi159.stdout:Created osd(s) 9 on host 'smithi159' 2023-03-27T10:21:02.768 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:02.769+0000 7fd9523c3700 1 -- 172.21.15.159:0/3069999361 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd93409b3a0 msgr2=0x7fd93409d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:02.768 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:02.769+0000 7fd9523c3700 1 --2- 172.21.15.159:0/3069999361 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd93409b3a0 0x7fd93409d850 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fd93c005fd0 tx=0x7fd93c005dc0 comp rx=0 tx=0).stop 2023-03-27T10:21:02.769 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:02.769+0000 7fd9523c3700 1 -- 172.21.15.159:0/3069999361 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd94c100830 msgr2=0x7fd94c1017c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:02.769 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:02.769+0000 7fd9523c3700 1 --2- 172.21.15.159:0/3069999361 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd94c100830 0x7fd94c1017c0 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7fd94400b970 tx=0x7fd94400bd30 comp rx=0 tx=0).stop 2023-03-27T10:21:02.769 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:02.769+0000 7fd9523c3700 1 -- 172.21.15.159:0/3069999361 shutdown_connections 2023-03-27T10:21:02.769 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:02.769+0000 7fd9523c3700 1 --2- 172.21.15.159:0/3069999361 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd93409b3a0 0x7fd93409d850 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:02.769 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:02.769+0000 7fd9523c3700 1 --2- 172.21.15.159:0/3069999361 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd94c0feca0 0x7fd94c102c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:02.769 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:02.769+0000 7fd9523c3700 1 --2- 172.21.15.159:0/3069999361 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd94c100830 0x7fd94c1017c0 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:02.769 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:02.769+0000 7fd9523c3700 1 --2- 172.21.15.159:0/3069999361 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd94c0ffea0 0x7fd94c101280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:02.770 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:02.769+0000 7fd9523c3700 1 -- 172.21.15.159:0/3069999361 >> 172.21.15.159:0/3069999361 conn(0x7fd94c0fa2b0 msgr2=0x7fd94c068120 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:21:02.770 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:02.769+0000 7fd9523c3700 1 -- 172.21.15.159:0/3069999361 shutdown_connections 2023-03-27T10:21:02.770 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:02.769+0000 7fd9523c3700 1 -- 172.21.15.159:0/3069999361 wait complete. 2023-03-27T10:21:02.776 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:02 smithi159 ceph-mon[93039]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:21:02.777 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:02 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:02.777 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:02 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:02.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:02 smithi161 ceph-mon[94333]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:21:02.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:02 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:02.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:02 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:03.189 DEBUG:teuthology.orchestra.run.smithi159:osd.9> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.9.service 2023-03-27T10:21:03.191 INFO:tasks.cephadm:Deploying osd.10 on smithi161 with /dev/vg_nvme/lv_1... 2023-03-27T10:21:03.192 DEBUG:teuthology.orchestra.run.smithi161:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-03-27T10:21:03.224 INFO:journalctl@ceph.osd.9.smithi159.stdout:-- Logs begin at Mon 2023-03-27 10:03:12 UTC. -- 2023-03-27T10:21:03.848 INFO:teuthology.orchestra.run.smithi161.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.b/config 2023-03-27T10:21:04.044 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:03 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:04.044 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:03 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:04.044 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:03 smithi159 ceph-mon[93039]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:21:04.044 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:03 smithi159 ceph-mon[93039]: from='osd.9 [v2:172.21.15.159:6826/3285179829,v1:172.21.15.159:6827/3285179829]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2023-03-27T10:21:04.045 INFO:journalctl@ceph.osd.9.smithi159.stdout:Mar 27 10:21:03 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-9[122365]: 2023-03-27T10:21:03.737+0000 7fba00bbe580 -1 osd.9 0 log_to_monitors true 2023-03-27T10:21:04.067 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:03 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:04.067 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:03 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:04.067 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:03 smithi161 ceph-mon[94333]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:21:04.067 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:03 smithi161 ceph-mon[94333]: from='osd.9 [v2:172.21.15.159:6826/3285179829,v1:172.21.15.159:6827/3285179829]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2023-03-27T10:21:04.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:04.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:03 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:04.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:03 smithi171 ceph-mon[94808]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:21:04.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:03 smithi171 ceph-mon[94808]: from='osd.9 [v2:172.21.15.159:6826/3285179829,v1:172.21.15.159:6827/3285179829]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2023-03-27T10:21:04.975 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:04 smithi161 ceph-mon[94333]: from='osd.9 [v2:172.21.15.159:6826/3285179829,v1:172.21.15.159:6827/3285179829]' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2023-03-27T10:21:04.975 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:04 smithi161 ceph-mon[94333]: osdmap e65: 10 total, 9 up, 10 in 2023-03-27T10:21:04.975 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:04 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:21:04.975 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:04 smithi161 ceph-mon[94333]: from='osd.9 [v2:172.21.15.159:6826/3285179829,v1:172.21.15.159:6827/3285179829]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2023-03-27T10:21:05.115 INFO:journalctl@ceph.osd.9.smithi159.stdout:Mar 27 10:21:04 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-9[122365]: 2023-03-27T10:21:04.776+0000 7fb9f6239700 -1 osd.9 0 waiting for initial osdmap 2023-03-27T10:21:05.115 INFO:journalctl@ceph.osd.9.smithi159.stdout:Mar 27 10:21:04 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-9[122365]: 2023-03-27T10:21:04.790+0000 7fb9f002a700 -1 osd.9 66 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-03-27T10:21:05.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:04 smithi159 ceph-mon[93039]: from='osd.9 [v2:172.21.15.159:6826/3285179829,v1:172.21.15.159:6827/3285179829]' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2023-03-27T10:21:05.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:04 smithi159 ceph-mon[93039]: osdmap e65: 10 total, 9 up, 10 in 2023-03-27T10:21:05.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:04 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:21:05.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:04 smithi159 ceph-mon[93039]: from='osd.9 [v2:172.21.15.159:6826/3285179829,v1:172.21.15.159:6827/3285179829]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2023-03-27T10:21:05.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:04 smithi171 ceph-mon[94808]: from='osd.9 [v2:172.21.15.159:6826/3285179829,v1:172.21.15.159:6827/3285179829]' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2023-03-27T10:21:05.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:04 smithi171 ceph-mon[94808]: osdmap e65: 10 total, 9 up, 10 in 2023-03-27T10:21:05.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:04 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:21:05.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:04 smithi171 ceph-mon[94808]: from='osd.9 [v2:172.21.15.159:6826/3285179829,v1:172.21.15.159:6827/3285179829]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2023-03-27T10:21:05.493 INFO:teuthology.orchestra.run.smithi161.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-03-27T10:21:05.493 INFO:teuthology.orchestra.run.smithi161.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-03-27T10:21:05.494 INFO:teuthology.orchestra.run.smithi161.stderr: stderr: 10+0 records in 2023-03-27T10:21:05.494 INFO:teuthology.orchestra.run.smithi161.stderr:10+0 records out 2023-03-27T10:21:05.494 INFO:teuthology.orchestra.run.smithi161.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0111334 s, 942 MB/s 2023-03-27T10:21:05.494 INFO:teuthology.orchestra.run.smithi161.stderr:--> Zapping successful for: 2023-03-27T10:21:05.784 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:05 smithi161 ceph-mon[94333]: from='osd.9 [v2:172.21.15.159:6826/3285179829,v1:172.21.15.159:6827/3285179829]' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2023-03-27T10:21:05.784 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:05 smithi161 ceph-mon[94333]: osdmap e66: 10 total, 9 up, 10 in 2023-03-27T10:21:05.889 DEBUG:teuthology.orchestra.run.smithi161:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch daemon add osd smithi161:vg_nvme/lv_1 2023-03-27T10:21:06.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:05 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:21:06.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:05 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:21:06.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:05 smithi161 ceph-mon[94333]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:21:06.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:05 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:06.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:05 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:06.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:05 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:06.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:05 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:06.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:05 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:06.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:05 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:06.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:05 smithi159 ceph-mon[93039]: from='osd.9 [v2:172.21.15.159:6826/3285179829,v1:172.21.15.159:6827/3285179829]' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2023-03-27T10:21:06.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:05 smithi159 ceph-mon[93039]: osdmap e66: 10 total, 9 up, 10 in 2023-03-27T10:21:06.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:05 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:21:06.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:05 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:21:06.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:05 smithi159 ceph-mon[93039]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:21:06.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:05 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:06.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:05 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:06.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:05 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:06.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:05 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:06.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:05 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:06.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:05 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:06.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:05 smithi171 ceph-mon[94808]: from='osd.9 [v2:172.21.15.159:6826/3285179829,v1:172.21.15.159:6827/3285179829]' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2023-03-27T10:21:06.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:05 smithi171 ceph-mon[94808]: osdmap e66: 10 total, 9 up, 10 in 2023-03-27T10:21:06.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:05 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:21:06.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:05 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:21:06.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:05 smithi171 ceph-mon[94808]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 53 MiB used, 805 GiB / 805 GiB avail 2023-03-27T10:21:06.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:05 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:06.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:05 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:06.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:05 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:06.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:05 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:06.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:05 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:06.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:05 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:06.533 INFO:teuthology.orchestra.run.smithi161.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.b/config 2023-03-27T10:21:07.074 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:06 smithi161 ceph-mon[94333]: purged_snaps scrub starts 2023-03-27T10:21:07.074 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:06 smithi161 ceph-mon[94333]: purged_snaps scrub ok 2023-03-27T10:21:07.074 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:06 smithi161 ceph-mon[94333]: Detected new or changed devices on smithi159 2023-03-27T10:21:07.074 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:06 smithi161 ceph-mon[94333]: Adjusting osd_memory_target on smithi159 to 4292M 2023-03-27T10:21:07.074 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:07.075 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:07.075 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:21:07.075 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:06 smithi161 ceph-mon[94333]: osd.9 [v2:172.21.15.159:6826/3285179829,v1:172.21.15.159:6827/3285179829] boot 2023-03-27T10:21:07.075 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:06 smithi161 ceph-mon[94333]: osdmap e67: 10 total, 10 up, 10 in 2023-03-27T10:21:07.075 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:07.075 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:21:07.075 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:07.075 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:07.076 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:07.076 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:07.076 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:07.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:06 smithi159 ceph-mon[93039]: purged_snaps scrub starts 2023-03-27T10:21:07.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:06 smithi159 ceph-mon[93039]: purged_snaps scrub ok 2023-03-27T10:21:07.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:06 smithi159 ceph-mon[93039]: Detected new or changed devices on smithi159 2023-03-27T10:21:07.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:06 smithi159 ceph-mon[93039]: Adjusting osd_memory_target on smithi159 to 4292M 2023-03-27T10:21:07.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:07.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:07.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:21:07.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:06 smithi159 ceph-mon[93039]: osd.9 [v2:172.21.15.159:6826/3285179829,v1:172.21.15.159:6827/3285179829] boot 2023-03-27T10:21:07.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:06 smithi159 ceph-mon[93039]: osdmap e67: 10 total, 10 up, 10 in 2023-03-27T10:21:07.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:07.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:21:07.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:07.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:07.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:07.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:07.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:07.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:06 smithi171 ceph-mon[94808]: purged_snaps scrub starts 2023-03-27T10:21:07.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:06 smithi171 ceph-mon[94808]: purged_snaps scrub ok 2023-03-27T10:21:07.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:06 smithi171 ceph-mon[94808]: Detected new or changed devices on smithi159 2023-03-27T10:21:07.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:06 smithi171 ceph-mon[94808]: Adjusting osd_memory_target on smithi159 to 4292M 2023-03-27T10:21:07.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:07.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:07.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:21:07.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:06 smithi171 ceph-mon[94808]: osd.9 [v2:172.21.15.159:6826/3285179829,v1:172.21.15.159:6827/3285179829] boot 2023-03-27T10:21:07.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:06 smithi171 ceph-mon[94808]: osdmap e67: 10 total, 10 up, 10 in 2023-03-27T10:21:07.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:07.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-03-27T10:21:07.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:07.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:07.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:07.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:07.228 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:07.775 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.774+0000 7fd14f80e700 1 -- 172.21.15.161:0/3363579787 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd148100830 msgr2=0x7fd1481070f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:07.775 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.774+0000 7fd14f80e700 1 --2- 172.21.15.161:0/3363579787 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd148100830 0x7fd1481070f0 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7fd14000b0a0 tx=0x7fd14000b3b0 comp rx=0 tx=0).stop 2023-03-27T10:21:07.775 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.775+0000 7fd14f80e700 1 -- 172.21.15.161:0/3363579787 shutdown_connections 2023-03-27T10:21:07.775 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.775+0000 7fd14f80e700 1 --2- 172.21.15.161:0/3363579787 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd148100830 0x7fd1481070f0 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:07.775 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.775+0000 7fd14f80e700 1 --2- 172.21.15.161:0/3363579787 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd1480feca0 0x7fd1480ff0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:07.776 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.775+0000 7fd14f80e700 1 --2- 172.21.15.161:0/3363579787 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd1480ffea0 0x7fd1481002f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:07.776 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.775+0000 7fd14f80e700 1 -- 172.21.15.161:0/3363579787 >> 172.21.15.161:0/3363579787 conn(0x7fd1480fa2d0 msgr2=0x7fd1480fc720 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:21:07.776 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.775+0000 7fd14f80e700 1 -- 172.21.15.161:0/3363579787 shutdown_connections 2023-03-27T10:21:07.776 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.775+0000 7fd14f80e700 1 -- 172.21.15.161:0/3363579787 wait complete. 2023-03-27T10:21:07.777 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.775+0000 7fd14f80e700 1 Processor -- start 2023-03-27T10:21:07.777 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.775+0000 7fd14f80e700 1 -- start start 2023-03-27T10:21:07.778 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14f80e700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd1480feca0 0x7fd148069470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:07.778 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14f80e700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd1480ffea0 0x7fd14819a8e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:07.778 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14f80e700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd148100830 0x7fd14819cf20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:07.778 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14f80e700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd14819d6c0 con 0x7fd148100830 2023-03-27T10:21:07.778 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14f80e700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd14819d830 con 0x7fd1480feca0 2023-03-27T10:21:07.778 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14f80e700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd14819d9f0 con 0x7fd1480ffea0 2023-03-27T10:21:07.778 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14d5aa700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd1480feca0 0x7fd148069470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:07.779 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14d5aa700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd1480feca0 0x7fd148069470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.161:3300/0 says I am v2:172.21.15.161:44820/0 (socket says 172.21.15.161:44820) 2023-03-27T10:21:07.779 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14d5aa700 1 -- 172.21.15.161:0/2997025997 learned_addr learned my addr 172.21.15.161:0/2997025997 (peer_addr_for_me v2:172.21.15.161:0/0) 2023-03-27T10:21:07.779 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14cda9700 1 --2- 172.21.15.161:0/2997025997 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd1480ffea0 0x7fd14819a8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:07.779 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14ddab700 1 --2- 172.21.15.161:0/2997025997 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd148100830 0x7fd14819cf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:07.779 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14d5aa700 1 -- 172.21.15.161:0/2997025997 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd1480ffea0 msgr2=0x7fd14819a8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:07.779 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14d5aa700 1 --2- 172.21.15.161:0/2997025997 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd1480ffea0 0x7fd14819a8e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:07.779 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14d5aa700 1 -- 172.21.15.161:0/2997025997 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd148100830 msgr2=0x7fd14819cf20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:07.779 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14d5aa700 1 --2- 172.21.15.161:0/2997025997 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd148100830 0x7fd14819cf20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:07.780 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14d5aa700 1 -- 172.21.15.161:0/2997025997 --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd140009d00 con 0x7fd1480feca0 2023-03-27T10:21:07.780 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14d5aa700 1 --2- 172.21.15.161:0/2997025997 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd1480feca0 0x7fd148069470 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7fd14400c970 tx=0x7fd14400cc80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:21:07.780 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd13e7fc700 1 -- 172.21.15.161:0/2997025997 <== mon.2 v2:172.21.15.161:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd144004220 con 0x7fd1480feca0 2023-03-27T10:21:07.780 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd13e7fc700 1 -- 172.21.15.161:0/2997025997 <== mon.2 v2:172.21.15.161:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd144007d60 con 0x7fd1480feca0 2023-03-27T10:21:07.780 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd13e7fc700 1 -- 172.21.15.161:0/2997025997 <== mon.2 v2:172.21.15.161:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd144003c90 con 0x7fd1480feca0 2023-03-27T10:21:07.780 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.776+0000 7fd14f80e700 1 -- 172.21.15.161:0/2997025997 --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd14810a600 con 0x7fd1480feca0 2023-03-27T10:21:07.780 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.777+0000 7fd14f80e700 1 -- 172.21.15.161:0/2997025997 --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd14810a470 con 0x7fd1480feca0 2023-03-27T10:21:07.780 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.777+0000 7fd14f80e700 1 -- 172.21.15.161:0/2997025997 --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd148106220 con 0x7fd1480feca0 2023-03-27T10:21:07.781 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.778+0000 7fd13e7fc700 1 -- 172.21.15.161:0/2997025997 <== mon.2 v2:172.21.15.161:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7fd14401f070 con 0x7fd1480feca0 2023-03-27T10:21:07.781 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.779+0000 7fd13e7fc700 1 --2- 172.21.15.161:0/2997025997 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd13409b3a0 0x7fd13409d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:07.781 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.779+0000 7fd13e7fc700 1 -- 172.21.15.161:0/2997025997 <== mon.2 v2:172.21.15.161:3300/0 5 ==== osd_map(68..68 src has 1..68) v4 ==== 6261+0+0 (secure 0 0 0) 0x7fd1440c4560 con 0x7fd1480feca0 2023-03-27T10:21:07.782 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.779+0000 7fd14cda9700 1 --2- 172.21.15.161:0/2997025997 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd13409b3a0 0x7fd13409d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:07.782 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.780+0000 7fd14cda9700 1 --2- 172.21.15.161:0/2997025997 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd13409b3a0 0x7fd13409d850 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7fd1480699f0 tx=0x7fd13800b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:21:07.782 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.782+0000 7fd13e7fc700 1 -- 172.21.15.161:0/2997025997 <== mon.2 v2:172.21.15.161:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fd1440906d0 con 0x7fd1480feca0 2023-03-27T10:21:07.977 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:07.977+0000 7fd14f80e700 1 -- 172.21.15.161:0/2997025997 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi161:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) v1 -- 0x7fd148061950 con 0x7fd13409b3a0 2023-03-27T10:21:08.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:07 smithi171 ceph-mon[94808]: osdmap e68: 10 total, 10 up, 10 in 2023-03-27T10:21:08.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:07 smithi171 ceph-mon[94808]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 59 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:08.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:07 smithi161 ceph-mon[94333]: osdmap e68: 10 total, 10 up, 10 in 2023-03-27T10:21:08.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:07 smithi161 ceph-mon[94333]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 59 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:08.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:07 smithi159 ceph-mon[93039]: osdmap e68: 10 total, 10 up, 10 in 2023-03-27T10:21:08.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:07 smithi159 ceph-mon[93039]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 59 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:09.223 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:08 smithi161 ceph-mon[94333]: from='client.24338 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi161:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:21:09.224 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:08 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:21:09.224 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:08 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:21:09.224 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:08 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:09.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:08 smithi171 ceph-mon[94808]: from='client.24338 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi161:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:21:09.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:21:09.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:21:09.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:09.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:08 smithi159 ceph-mon[93039]: from='client.24338 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi161:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:21:09.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:21:09.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:21:09.365 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:10.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:09 smithi161 ceph-mon[94333]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 59 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:10.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:09 smithi171 ceph-mon[94808]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 59 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:10.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:09 smithi159 ceph-mon[93039]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 59 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:11.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:11 smithi171 ceph-mon[94808]: from='client.? 172.21.15.161:0/3204380822' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4871ca03-30ae-4e87-942c-9441dd31fb40"}]: dispatch 2023-03-27T10:21:11.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:11 smithi171 ceph-mon[94808]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4871ca03-30ae-4e87-942c-9441dd31fb40"}]: dispatch 2023-03-27T10:21:11.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:11 smithi171 ceph-mon[94808]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4871ca03-30ae-4e87-942c-9441dd31fb40"}]': finished 2023-03-27T10:21:11.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:11 smithi171 ceph-mon[94808]: osdmap e69: 11 total, 10 up, 11 in 2023-03-27T10:21:11.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:11 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-03-27T10:21:11.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:11 smithi171 ceph-mon[94808]: from='client.? 172.21.15.161:0/617941866' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:21:11.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:11 smithi161 ceph-mon[94333]: from='client.? 172.21.15.161:0/3204380822' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4871ca03-30ae-4e87-942c-9441dd31fb40"}]: dispatch 2023-03-27T10:21:11.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:11 smithi161 ceph-mon[94333]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4871ca03-30ae-4e87-942c-9441dd31fb40"}]: dispatch 2023-03-27T10:21:11.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:11 smithi161 ceph-mon[94333]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4871ca03-30ae-4e87-942c-9441dd31fb40"}]': finished 2023-03-27T10:21:11.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:11 smithi161 ceph-mon[94333]: osdmap e69: 11 total, 10 up, 11 in 2023-03-27T10:21:11.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-03-27T10:21:11.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:11 smithi161 ceph-mon[94333]: from='client.? 172.21.15.161:0/617941866' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:21:11.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:11 smithi159 ceph-mon[93039]: from='client.? 172.21.15.161:0/3204380822' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4871ca03-30ae-4e87-942c-9441dd31fb40"}]: dispatch 2023-03-27T10:21:11.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:11 smithi159 ceph-mon[93039]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4871ca03-30ae-4e87-942c-9441dd31fb40"}]: dispatch 2023-03-27T10:21:11.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:11 smithi159 ceph-mon[93039]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4871ca03-30ae-4e87-942c-9441dd31fb40"}]': finished 2023-03-27T10:21:11.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:11 smithi159 ceph-mon[93039]: osdmap e69: 11 total, 10 up, 11 in 2023-03-27T10:21:11.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:11 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-03-27T10:21:11.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:11 smithi159 ceph-mon[93039]: from='client.? 172.21.15.161:0/617941866' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:21:12.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:12 smithi161 ceph-mon[94333]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:12.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:12 smithi159 ceph-mon[93039]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:12.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:12 smithi171 ceph-mon[94808]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:14.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:14 smithi161 ceph-mon[94333]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:14.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:14 smithi159 ceph-mon[93039]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:14.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:14 smithi171 ceph-mon[94808]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:16.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:16 smithi161 ceph-mon[94333]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:16.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:16 smithi159 ceph-mon[93039]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:16.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:16 smithi171 ceph-mon[94808]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:17.820 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:17 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2023-03-27T10:21:17.820 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:17 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:17.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:17 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2023-03-27T10:21:17.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:17 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:17.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:17 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2023-03-27T10:21:17.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:17 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:18.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:18 smithi161 ceph-mon[94333]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:18.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:18 smithi161 ceph-mon[94333]: Deploying daemon osd.10 on smithi161 2023-03-27T10:21:18.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:18 smithi159 ceph-mon[93039]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:18.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:18 smithi159 ceph-mon[93039]: Deploying daemon osd.10 on smithi161 2023-03-27T10:21:18.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:18 smithi171 ceph-mon[94808]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:18.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:18 smithi171 ceph-mon[94808]: Deploying daemon osd.10 on smithi161 2023-03-27T10:21:20.813 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:20 smithi161 ceph-mon[94333]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:20.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:20 smithi159 ceph-mon[93039]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:20.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:20 smithi171 ceph-mon[94808]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:22.774 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:22 smithi161 ceph-mon[94333]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:22.774 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:22 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:22.774 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:22 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:22.774 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:22 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:22.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:22 smithi159 ceph-mon[93039]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:22.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:22 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:22.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:22 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:22.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:22 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:22.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:22 smithi171 ceph-mon[94808]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:22.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:22 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:22.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:22 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:22.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:22 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:24.539 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:24 smithi161 ceph-mon[94333]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:24.540 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:24.540 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:24.637 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:24.638+0000 7fd13e7fc700 1 -- 172.21.15.161:0/2997025997 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+38 (secure 0 0 0) 0x7fd148061950 con 0x7fd13409b3a0 2023-03-27T10:21:24.662 INFO:teuthology.orchestra.run.smithi161.stdout:Created osd(s) 10 on host 'smithi161' 2023-03-27T10:21:24.662 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:24.640+0000 7fd14f80e700 1 -- 172.21.15.161:0/2997025997 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd13409b3a0 msgr2=0x7fd13409d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:24.663 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:24.640+0000 7fd14f80e700 1 --2- 172.21.15.161:0/2997025997 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd13409b3a0 0x7fd13409d850 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7fd1480699f0 tx=0x7fd13800b410 comp rx=0 tx=0).stop 2023-03-27T10:21:24.663 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:24.640+0000 7fd14f80e700 1 -- 172.21.15.161:0/2997025997 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd1480feca0 msgr2=0x7fd148069470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:24.663 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:24.640+0000 7fd14f80e700 1 --2- 172.21.15.161:0/2997025997 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd1480feca0 0x7fd148069470 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7fd14400c970 tx=0x7fd14400cc80 comp rx=0 tx=0).stop 2023-03-27T10:21:24.663 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:24.640+0000 7fd14f80e700 1 -- 172.21.15.161:0/2997025997 shutdown_connections 2023-03-27T10:21:24.663 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:24.640+0000 7fd14f80e700 1 --2- 172.21.15.161:0/2997025997 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd13409b3a0 0x7fd13409d850 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:24.663 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:24.640+0000 7fd14f80e700 1 --2- 172.21.15.161:0/2997025997 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd148100830 0x7fd14819cf20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:24.663 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:24.640+0000 7fd14f80e700 1 --2- 172.21.15.161:0/2997025997 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd1480feca0 0x7fd148069470 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:24.664 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:24.640+0000 7fd14f80e700 1 --2- 172.21.15.161:0/2997025997 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd1480ffea0 0x7fd14819a8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:24.664 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:24.640+0000 7fd14f80e700 1 -- 172.21.15.161:0/2997025997 >> 172.21.15.161:0/2997025997 conn(0x7fd1480fa2d0 msgr2=0x7fd148103360 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:21:24.664 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:24.640+0000 7fd14f80e700 1 -- 172.21.15.161:0/2997025997 shutdown_connections 2023-03-27T10:21:24.664 INFO:teuthology.orchestra.run.smithi161.stderr:2023-03-27T10:21:24.640+0000 7fd14f80e700 1 -- 172.21.15.161:0/2997025997 wait complete. 2023-03-27T10:21:24.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:24 smithi159 ceph-mon[93039]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:24.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:24.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:24.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:24 smithi171 ceph-mon[94808]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:24.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:24.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:25.130 DEBUG:teuthology.orchestra.run.smithi161:osd.10> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.10.service 2023-03-27T10:21:25.132 INFO:tasks.cephadm:Deploying osd.11 on smithi171 with /dev/vg_nvme/lv_1... 2023-03-27T10:21:25.132 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-03-27T10:21:25.161 INFO:journalctl@ceph.osd.10.smithi161.stdout:-- Logs begin at Mon 2023-03-27 10:02:59 UTC. -- 2023-03-27T10:21:25.334 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:21:25 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:21:25.285+0000 7feccaef7580 -1 osd.10 0 log_to_monitors true 2023-03-27T10:21:25.783 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.c/config 2023-03-27T10:21:25.886 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:25 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:25.886 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:25 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:25.886 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:25 smithi161 ceph-mon[94333]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:25.886 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:25 smithi161 ceph-mon[94333]: from='osd.10 [v2:172.21.15.161:6824/946677932,v1:172.21.15.161:6825/946677932]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2023-03-27T10:21:25.886 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:25 smithi161 ceph-mon[94333]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2023-03-27T10:21:25.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:25 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:25.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:25 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:25.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:25 smithi171 ceph-mon[94808]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:25.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:25 smithi171 ceph-mon[94808]: from='osd.10 [v2:172.21.15.161:6824/946677932,v1:172.21.15.161:6825/946677932]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2023-03-27T10:21:25.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:25 smithi171 ceph-mon[94808]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2023-03-27T10:21:26.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:25 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:26.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:25 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:26.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:25 smithi159 ceph-mon[93039]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:26.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:25 smithi159 ceph-mon[93039]: from='osd.10 [v2:172.21.15.161:6824/946677932,v1:172.21.15.161:6825/946677932]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2023-03-27T10:21:26.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:25 smithi159 ceph-mon[93039]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2023-03-27T10:21:26.736 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:21:26 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:21:26.652+0000 7fecc0572700 -1 osd.10 0 waiting for initial osdmap 2023-03-27T10:21:26.736 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:21:26 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:21:26.665+0000 7fecb835f700 -1 osd.10 71 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-03-27T10:21:26.737 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:26 smithi161 ceph-mon[94333]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2023-03-27T10:21:26.737 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:26 smithi161 ceph-mon[94333]: osdmap e70: 11 total, 10 up, 11 in 2023-03-27T10:21:26.737 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:26 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-03-27T10:21:26.737 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:26 smithi161 ceph-mon[94333]: from='osd.10 [v2:172.21.15.161:6824/946677932,v1:172.21.15.161:6825/946677932]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:21:26.737 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:26 smithi161 ceph-mon[94333]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:21:26.910 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:26 smithi171 ceph-mon[94808]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2023-03-27T10:21:26.910 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:26 smithi171 ceph-mon[94808]: osdmap e70: 11 total, 10 up, 11 in 2023-03-27T10:21:26.910 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:26 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-03-27T10:21:26.910 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:26 smithi171 ceph-mon[94808]: from='osd.10 [v2:172.21.15.161:6824/946677932,v1:172.21.15.161:6825/946677932]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:21:26.910 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:26 smithi171 ceph-mon[94808]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:21:27.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:26 smithi159 ceph-mon[93039]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2023-03-27T10:21:27.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:26 smithi159 ceph-mon[93039]: osdmap e70: 11 total, 10 up, 11 in 2023-03-27T10:21:27.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:26 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-03-27T10:21:27.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:26 smithi159 ceph-mon[93039]: from='osd.10 [v2:172.21.15.161:6824/946677932,v1:172.21.15.161:6825/946677932]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:21:27.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:26 smithi159 ceph-mon[93039]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]: dispatch 2023-03-27T10:21:27.404 INFO:teuthology.orchestra.run.smithi171.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-03-27T10:21:27.404 INFO:teuthology.orchestra.run.smithi171.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-03-27T10:21:27.405 INFO:teuthology.orchestra.run.smithi171.stderr: stderr: 10+0 records in 2023-03-27T10:21:27.405 INFO:teuthology.orchestra.run.smithi171.stderr:10+0 records out 2023-03-27T10:21:27.405 INFO:teuthology.orchestra.run.smithi171.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0111815 s, 938 MB/s 2023-03-27T10:21:27.405 INFO:teuthology.orchestra.run.smithi171.stderr: stderr: 2023-03-27T10:21:27.405 INFO:teuthology.orchestra.run.smithi171.stderr:--> Zapping successful for: 2023-03-27T10:21:27.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:27 smithi171 ceph-mon[94808]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]': finished 2023-03-27T10:21:27.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:27 smithi171 ceph-mon[94808]: osdmap e71: 11 total, 10 up, 11 in 2023-03-27T10:21:27.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:27 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-03-27T10:21:27.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:27 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-03-27T10:21:27.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:27 smithi171 ceph-mon[94808]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:27.842 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch daemon add osd smithi171:vg_nvme/lv_1 2023-03-27T10:21:28.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:27 smithi161 ceph-mon[94333]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]': finished 2023-03-27T10:21:28.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:27 smithi161 ceph-mon[94333]: osdmap e71: 11 total, 10 up, 11 in 2023-03-27T10:21:28.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:27 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-03-27T10:21:28.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:27 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-03-27T10:21:28.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:27 smithi161 ceph-mon[94333]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:28.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:27 smithi159 ceph-mon[93039]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi161", "root=default"]}]': finished 2023-03-27T10:21:28.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:27 smithi159 ceph-mon[93039]: osdmap e71: 11 total, 10 up, 11 in 2023-03-27T10:21:28.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:27 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-03-27T10:21:28.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:27 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-03-27T10:21:28.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:27 smithi159 ceph-mon[93039]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 60 MiB used, 894 GiB / 894 GiB avail 2023-03-27T10:21:28.483 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.c/config 2023-03-27T10:21:28.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: purged_snaps scrub starts 2023-03-27T10:21:28.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: purged_snaps scrub ok 2023-03-27T10:21:28.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: osd.10 [v2:172.21.15.161:6824/946677932,v1:172.21.15.161:6825/946677932] boot 2023-03-27T10:21:28.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: osdmap e72: 11 total, 11 up, 11 in 2023-03-27T10:21:28.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-03-27T10:21:28.979 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: Detected new or changed devices on smithi161 2023-03-27T10:21:28.979 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:28.979 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:28.979 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:28.980 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:28.980 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:28.980 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:28.980 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: Adjusting osd_memory_target on smithi161 to 4292M 2023-03-27T10:21:28.980 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:28.981 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:28.981 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:28.981 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:28.982 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:28.982 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:28.982 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:28.982 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:28 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:29.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: purged_snaps scrub starts 2023-03-27T10:21:29.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: purged_snaps scrub ok 2023-03-27T10:21:29.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: osd.10 [v2:172.21.15.161:6824/946677932,v1:172.21.15.161:6825/946677932] boot 2023-03-27T10:21:29.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: osdmap e72: 11 total, 11 up, 11 in 2023-03-27T10:21:29.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-03-27T10:21:29.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: Detected new or changed devices on smithi161 2023-03-27T10:21:29.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:29.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:29.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:29.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:29.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:29.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:29.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: Adjusting osd_memory_target on smithi161 to 4292M 2023-03-27T10:21:29.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:29.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:29.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:29.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:29.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:29.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:29.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:29.087 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:28 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:29.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: purged_snaps scrub starts 2023-03-27T10:21:29.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: purged_snaps scrub ok 2023-03-27T10:21:29.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: osd.10 [v2:172.21.15.161:6824/946677932,v1:172.21.15.161:6825/946677932] boot 2023-03-27T10:21:29.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: osdmap e72: 11 total, 11 up, 11 in 2023-03-27T10:21:29.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-03-27T10:21:29.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: Detected new or changed devices on smithi161 2023-03-27T10:21:29.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:29.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:29.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:29.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:29.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:29.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:29.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: Adjusting osd_memory_target on smithi161 to 4292M 2023-03-27T10:21:29.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:29.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:29.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:29.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:29.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:29.117 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:29.118 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:29.118 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:28 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:29.644 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.644+0000 7f2a9edd0700 1 -- 172.21.15.171:0/1231668716 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f2a980ffea0 msgr2=0x7f2a981002f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:29.644 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.644+0000 7f2a9edd0700 1 --2- 172.21.15.171:0/1231668716 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f2a980ffea0 0x7f2a981002f0 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f2a84009990 tx=0x7f2a84009ca0 comp rx=0 tx=0).stop 2023-03-27T10:21:29.645 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.644+0000 7f2a9edd0700 1 -- 172.21.15.171:0/1231668716 shutdown_connections 2023-03-27T10:21:29.645 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.644+0000 7f2a9edd0700 1 --2- 172.21.15.171:0/1231668716 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f2a98100830 0x7f2a981070f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:29.645 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.644+0000 7f2a9edd0700 1 --2- 172.21.15.171:0/1231668716 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2a980feca0 0x7f2a980ff0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:29.645 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.644+0000 7f2a9edd0700 1 --2- 172.21.15.171:0/1231668716 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f2a980ffea0 0x7f2a981002f0 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:29.645 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.644+0000 7f2a9edd0700 1 -- 172.21.15.171:0/1231668716 >> 172.21.15.171:0/1231668716 conn(0x7f2a980fa2b0 msgr2=0x7f2a980fc720 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:21:29.645 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.644+0000 7f2a9edd0700 1 -- 172.21.15.171:0/1231668716 shutdown_connections 2023-03-27T10:21:29.646 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.644+0000 7f2a9edd0700 1 -- 172.21.15.171:0/1231668716 wait complete. 2023-03-27T10:21:29.646 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.645+0000 7f2a9edd0700 1 Processor -- start 2023-03-27T10:21:29.646 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.645+0000 7f2a9edd0700 1 -- start start 2023-03-27T10:21:29.646 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.645+0000 7f2a9edd0700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f2a980feca0 0x7f2a98068c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:29.646 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.645+0000 7f2a9edd0700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f2a980ffea0 0x7f2a9819a8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:29.646 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.645+0000 7f2a9edd0700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2a98100830 0x7f2a9819cf00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:29.646 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.645+0000 7f2a9edd0700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a9819d6a0 con 0x7f2a98100830 2023-03-27T10:21:29.647 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.645+0000 7f2a9edd0700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a9819d810 con 0x7f2a980ffea0 2023-03-27T10:21:29.647 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.645+0000 7f2a9edd0700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a9819d9d0 con 0x7f2a980feca0 2023-03-27T10:21:29.647 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.645+0000 7f2a9cb6c700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f2a980feca0 0x7f2a98068c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:29.647 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.645+0000 7f2a9cb6c700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f2a980feca0 0x7f2a98068c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.171:3300/0 says I am v2:172.21.15.171:55472/0 (socket says 172.21.15.171:55472) 2023-03-27T10:21:29.647 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.645+0000 7f2a9cb6c700 1 -- 172.21.15.171:0/3605763207 learned_addr learned my addr 172.21.15.171:0/3605763207 (peer_addr_for_me v2:172.21.15.171:0/0) 2023-03-27T10:21:29.647 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.645+0000 7f2a97fff700 1 --2- 172.21.15.171:0/3605763207 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f2a980ffea0 0x7f2a9819a8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:29.647 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.645+0000 7f2a9d36d700 1 --2- 172.21.15.171:0/3605763207 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2a98100830 0x7f2a9819cf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:29.648 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.646+0000 7f2a9cb6c700 1 -- 172.21.15.171:0/3605763207 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f2a980ffea0 msgr2=0x7f2a9819a8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:29.648 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.646+0000 7f2a9cb6c700 1 --2- 172.21.15.171:0/3605763207 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f2a980ffea0 0x7f2a9819a8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:29.648 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.646+0000 7f2a9cb6c700 1 -- 172.21.15.171:0/3605763207 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2a98100830 msgr2=0x7f2a9819cf00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:29.648 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.646+0000 7f2a9cb6c700 1 --2- 172.21.15.171:0/3605763207 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2a98100830 0x7f2a9819cf00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:29.648 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.646+0000 7f2a9cb6c700 1 -- 172.21.15.171:0/3605763207 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2a840095f0 con 0x7f2a980feca0 2023-03-27T10:21:29.648 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.646+0000 7f2a9cb6c700 1 --2- 172.21.15.171:0/3605763207 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f2a980feca0 0x7f2a98068c00 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f2a8c00b500 tx=0x7f2a8c00b810 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:21:29.649 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.646+0000 7f2a95ffb700 1 -- 172.21.15.171:0/3605763207 <== mon.1 v2:172.21.15.171:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a8c011830 con 0x7f2a980feca0 2023-03-27T10:21:29.650 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.646+0000 7f2a95ffb700 1 -- 172.21.15.171:0/3605763207 <== mon.1 v2:172.21.15.171:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2a8c011e50 con 0x7f2a980feca0 2023-03-27T10:21:29.650 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.646+0000 7f2a9edd0700 1 -- 172.21.15.171:0/3605763207 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2a9810a690 con 0x7f2a980feca0 2023-03-27T10:21:29.650 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.646+0000 7f2a95ffb700 1 -- 172.21.15.171:0/3605763207 <== mon.1 v2:172.21.15.171:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a8c00f2e0 con 0x7f2a980feca0 2023-03-27T10:21:29.650 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.646+0000 7f2a9edd0700 1 -- 172.21.15.171:0/3605763207 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2a9810a500 con 0x7f2a980feca0 2023-03-27T10:21:29.650 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.647+0000 7f2a9edd0700 1 -- 172.21.15.171:0/3605763207 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2a981062c0 con 0x7f2a980feca0 2023-03-27T10:21:29.650 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.648+0000 7f2a95ffb700 1 -- 172.21.15.171:0/3605763207 <== mon.1 v2:172.21.15.171:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f2a8c021070 con 0x7f2a980feca0 2023-03-27T10:21:29.650 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.648+0000 7f2a95ffb700 1 --2- 172.21.15.171:0/3605763207 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f2a8809b3a0 0x7f2a8809d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:29.651 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.648+0000 7f2a95ffb700 1 -- 172.21.15.171:0/3605763207 <== mon.1 v2:172.21.15.171:3300/0 5 ==== osd_map(73..73 src has 1..73) v4 ==== 6723+0+0 (secure 0 0 0) 0x7f2a8c0c42a0 con 0x7f2a980feca0 2023-03-27T10:21:29.651 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.649+0000 7f2a97fff700 1 --2- 172.21.15.171:0/3605763207 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f2a8809b3a0 0x7f2a8809d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:29.651 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.649+0000 7f2a97fff700 1 --2- 172.21.15.171:0/3605763207 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f2a8809b3a0 0x7f2a8809d850 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f2a98069180 tx=0x7f2a84005fd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:21:29.651 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.653+0000 7f2a95ffb700 1 -- 172.21.15.171:0/3605763207 <== mon.1 v2:172.21.15.171:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f2a8c090150 con 0x7f2a980feca0 2023-03-27T10:21:29.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:29 smithi171 ceph-mon[94808]: osdmap e73: 11 total, 11 up, 11 in 2023-03-27T10:21:29.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:29 smithi171 ceph-mon[94808]: pgmap v214: 1 pgs: 1 remapped+peering; 577 KiB data, 66 MiB used, 983 GiB / 983 GiB avail 2023-03-27T10:21:29.844 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:29.845+0000 7f2a9edd0700 1 -- 172.21.15.171:0/3605763207 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) v1 -- 0x7f2a98061970 con 0x7f2a8809b3a0 2023-03-27T10:21:30.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:29 smithi161 ceph-mon[94333]: osdmap e73: 11 total, 11 up, 11 in 2023-03-27T10:21:30.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:29 smithi161 ceph-mon[94333]: pgmap v214: 1 pgs: 1 remapped+peering; 577 KiB data, 66 MiB used, 983 GiB / 983 GiB avail 2023-03-27T10:21:30.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:29 smithi159 ceph-mon[93039]: osdmap e73: 11 total, 11 up, 11 in 2023-03-27T10:21:30.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:29 smithi159 ceph-mon[93039]: pgmap v214: 1 pgs: 1 remapped+peering; 577 KiB data, 66 MiB used, 983 GiB / 983 GiB avail 2023-03-27T10:21:30.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:30 smithi171 ceph-mon[94808]: osdmap e74: 11 total, 11 up, 11 in 2023-03-27T10:21:30.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:30 smithi171 ceph-mon[94808]: from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:21:30.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:21:30.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:21:30.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:30 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:30.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:30 smithi171 ceph-mon[94808]: osdmap e75: 11 total, 11 up, 11 in 2023-03-27T10:21:31.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:30 smithi161 ceph-mon[94333]: osdmap e74: 11 total, 11 up, 11 in 2023-03-27T10:21:31.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:30 smithi161 ceph-mon[94333]: from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:21:31.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:21:31.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:21:31.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:30 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:31.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:30 smithi161 ceph-mon[94333]: osdmap e75: 11 total, 11 up, 11 in 2023-03-27T10:21:31.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:30 smithi159 ceph-mon[93039]: osdmap e74: 11 total, 11 up, 11 in 2023-03-27T10:21:31.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:30 smithi159 ceph-mon[93039]: from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:21:31.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-03-27T10:21:31.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-03-27T10:21:31.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:30 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:31.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:30 smithi159 ceph-mon[93039]: osdmap e75: 11 total, 11 up, 11 in 2023-03-27T10:21:32.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:32 smithi171 ceph-mon[94808]: pgmap v217: 1 pgs: 1 remapped+peering; 577 KiB data, 66 MiB used, 983 GiB / 983 GiB avail 2023-03-27T10:21:32.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:32 smithi171 ceph-mon[94808]: osdmap e76: 11 total, 11 up, 11 in 2023-03-27T10:21:32.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:32 smithi171 ceph-mon[94808]: from='client.? 172.21.15.171:0/958373538' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1fb37d9a-ed52-4c0d-a082-65e17d242106"}]: dispatch 2023-03-27T10:21:32.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:32 smithi171 ceph-mon[94808]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1fb37d9a-ed52-4c0d-a082-65e17d242106"}]: dispatch 2023-03-27T10:21:32.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:32 smithi171 ceph-mon[94808]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1fb37d9a-ed52-4c0d-a082-65e17d242106"}]': finished 2023-03-27T10:21:32.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:32 smithi171 ceph-mon[94808]: osdmap e77: 12 total, 11 up, 12 in 2023-03-27T10:21:32.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:32 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:32.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:32 smithi161 ceph-mon[94333]: pgmap v217: 1 pgs: 1 remapped+peering; 577 KiB data, 66 MiB used, 983 GiB / 983 GiB avail 2023-03-27T10:21:32.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:32 smithi161 ceph-mon[94333]: osdmap e76: 11 total, 11 up, 11 in 2023-03-27T10:21:32.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:32 smithi161 ceph-mon[94333]: from='client.? 172.21.15.171:0/958373538' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1fb37d9a-ed52-4c0d-a082-65e17d242106"}]: dispatch 2023-03-27T10:21:32.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:32 smithi161 ceph-mon[94333]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1fb37d9a-ed52-4c0d-a082-65e17d242106"}]: dispatch 2023-03-27T10:21:32.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:32 smithi161 ceph-mon[94333]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1fb37d9a-ed52-4c0d-a082-65e17d242106"}]': finished 2023-03-27T10:21:32.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:32 smithi161 ceph-mon[94333]: osdmap e77: 12 total, 11 up, 12 in 2023-03-27T10:21:32.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:32 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:32.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:32 smithi159 ceph-mon[93039]: pgmap v217: 1 pgs: 1 remapped+peering; 577 KiB data, 66 MiB used, 983 GiB / 983 GiB avail 2023-03-27T10:21:32.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:32 smithi159 ceph-mon[93039]: osdmap e76: 11 total, 11 up, 11 in 2023-03-27T10:21:32.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:32 smithi159 ceph-mon[93039]: from='client.? 172.21.15.171:0/958373538' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1fb37d9a-ed52-4c0d-a082-65e17d242106"}]: dispatch 2023-03-27T10:21:32.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:32 smithi159 ceph-mon[93039]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1fb37d9a-ed52-4c0d-a082-65e17d242106"}]: dispatch 2023-03-27T10:21:32.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:32 smithi159 ceph-mon[93039]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1fb37d9a-ed52-4c0d-a082-65e17d242106"}]': finished 2023-03-27T10:21:32.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:32 smithi159 ceph-mon[93039]: osdmap e77: 12 total, 11 up, 12 in 2023-03-27T10:21:32.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:32 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:33.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:33 smithi171 ceph-mon[94808]: from='client.? 172.21.15.171:0/1760477481' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:21:33.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:33 smithi161 ceph-mon[94333]: from='client.? 172.21.15.171:0/1760477481' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:21:33.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:33 smithi159 ceph-mon[93039]: from='client.? 172.21.15.171:0/1760477481' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-03-27T10:21:34.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:34 smithi171 ceph-mon[94808]: pgmap v220: 1 pgs: 1 remapped+peering; 577 KiB data, 66 MiB used, 983 GiB / 983 GiB avail 2023-03-27T10:21:34.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:34 smithi161 ceph-mon[94333]: pgmap v220: 1 pgs: 1 remapped+peering; 577 KiB data, 66 MiB used, 983 GiB / 983 GiB avail 2023-03-27T10:21:34.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:34 smithi159 ceph-mon[93039]: pgmap v220: 1 pgs: 1 remapped+peering; 577 KiB data, 66 MiB used, 983 GiB / 983 GiB avail 2023-03-27T10:21:36.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:36 smithi171 ceph-mon[94808]: pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 108 KiB/s, 0 objects/s recovering 2023-03-27T10:21:36.833 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:36 smithi161 ceph-mon[94333]: pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 108 KiB/s, 0 objects/s recovering 2023-03-27T10:21:36.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:36 smithi159 ceph-mon[93039]: pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 108 KiB/s, 0 objects/s recovering 2023-03-27T10:21:38.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:38 smithi171 ceph-mon[94808]: pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-03-27T10:21:38.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:38 smithi161 ceph-mon[94333]: pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-03-27T10:21:38.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:38 smithi159 ceph-mon[93039]: pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-03-27T10:21:39.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:39 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2023-03-27T10:21:39.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:39 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:39.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:39 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2023-03-27T10:21:39.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:39 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:39.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:39 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2023-03-27T10:21:39.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:39 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:40.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:40 smithi171 ceph-mon[94808]: Deploying daemon osd.11 on smithi171 2023-03-27T10:21:40.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:40 smithi171 ceph-mon[94808]: pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-03-27T10:21:40.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:40 smithi161 ceph-mon[94333]: Deploying daemon osd.11 on smithi171 2023-03-27T10:21:40.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:40 smithi161 ceph-mon[94333]: pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-03-27T10:21:40.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:40 smithi159 ceph-mon[93039]: Deploying daemon osd.11 on smithi171 2023-03-27T10:21:40.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:40 smithi159 ceph-mon[93039]: pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-03-27T10:21:42.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:42 smithi171 ceph-mon[94808]: pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-03-27T10:21:42.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:42 smithi161 ceph-mon[94333]: pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-03-27T10:21:42.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:42 smithi159 ceph-mon[93039]: pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-03-27T10:21:44.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:44 smithi171 ceph-mon[94808]: pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-03-27T10:21:44.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:44.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:44.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:44 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:44.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:44 smithi161 ceph-mon[94333]: pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-03-27T10:21:44.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:44.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:44.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:44 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:44.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:44 smithi159 ceph-mon[93039]: pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-03-27T10:21:44.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:44.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:44.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:44 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:46.143 INFO:teuthology.orchestra.run.smithi171.stdout:Created osd(s) 11 on host 'smithi171' 2023-03-27T10:21:46.143 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:46.143+0000 7f2a95ffb700 1 -- 172.21.15.171:0/3605763207 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+38 (secure 0 0 0) 0x7f2a98061970 con 0x7f2a8809b3a0 2023-03-27T10:21:46.145 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:46.146+0000 7f2a9edd0700 1 -- 172.21.15.171:0/3605763207 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f2a8809b3a0 msgr2=0x7f2a8809d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:46.145 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:46.146+0000 7f2a9edd0700 1 --2- 172.21.15.171:0/3605763207 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f2a8809b3a0 0x7f2a8809d850 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f2a98069180 tx=0x7f2a84005fd0 comp rx=0 tx=0).stop 2023-03-27T10:21:46.145 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:46.146+0000 7f2a9edd0700 1 -- 172.21.15.171:0/3605763207 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f2a980feca0 msgr2=0x7f2a98068c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:46.146 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:46.146+0000 7f2a9edd0700 1 --2- 172.21.15.171:0/3605763207 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f2a980feca0 0x7f2a98068c00 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f2a8c00b500 tx=0x7f2a8c00b810 comp rx=0 tx=0).stop 2023-03-27T10:21:46.146 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:46.146+0000 7f2a9edd0700 1 -- 172.21.15.171:0/3605763207 shutdown_connections 2023-03-27T10:21:46.146 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:46.146+0000 7f2a9edd0700 1 --2- 172.21.15.171:0/3605763207 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f2a8809b3a0 0x7f2a8809d850 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:46.146 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:46.146+0000 7f2a9edd0700 1 --2- 172.21.15.171:0/3605763207 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f2a980feca0 0x7f2a98068c00 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:46.146 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:46.146+0000 7f2a9edd0700 1 --2- 172.21.15.171:0/3605763207 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f2a98100830 0x7f2a9819cf00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:46.146 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:46.146+0000 7f2a9edd0700 1 --2- 172.21.15.171:0/3605763207 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f2a980ffea0 0x7f2a9819a8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:46.147 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:46.146+0000 7f2a9edd0700 1 -- 172.21.15.171:0/3605763207 >> 172.21.15.171:0/3605763207 conn(0x7f2a980fa2b0 msgr2=0x7f2a98103400 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:21:46.147 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:46.146+0000 7f2a9edd0700 1 -- 172.21.15.171:0/3605763207 shutdown_connections 2023-03-27T10:21:46.147 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:46.146+0000 7f2a9edd0700 1 -- 172.21.15.171:0/3605763207 wait complete. 2023-03-27T10:21:46.569 DEBUG:teuthology.orchestra.run.smithi171:osd.11> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.11.service 2023-03-27T10:21:46.571 INFO:tasks.cephadm:Waiting for 12 OSDs to come up... 2023-03-27T10:21:46.572 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph osd stat -f json 2023-03-27T10:21:46.601 INFO:journalctl@ceph.osd.11.smithi171.stdout:-- Logs begin at Mon 2023-03-27 10:03:15 UTC. -- 2023-03-27T10:21:46.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:46 smithi171 ceph-mon[94808]: pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-03-27T10:21:46.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:46 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:46.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:46 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:46.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:46 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:46.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:46 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:46.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:46 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:46.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:46 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:46.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:46 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:46.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:46 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:46.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:46 smithi161 ceph-mon[94333]: pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-03-27T10:21:46.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:46 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:46.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:46 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:46.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:46 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:46.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:46 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:46.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:46 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:46.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:46 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:46.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:46 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:46.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:46 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:46.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:46 smithi159 ceph-mon[93039]: pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-03-27T10:21:46.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:46 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:46.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:46 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:46.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:46 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:46.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:46 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:46.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:46 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:46.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:46 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:46.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:46 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:46.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:46 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:21:47.227 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:21:47.369 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:21:47 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:21:47.257+0000 7f40534b7580 -1 osd.11 0 log_to_monitors true 2023-03-27T10:21:47.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:47 smithi171 ceph-mon[94808]: from='osd.11 [v2:172.21.15.171:6824/596725174,v1:172.21.15.171:6825/596725174]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2023-03-27T10:21:47.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:47 smithi171 ceph-mon[94808]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2023-03-27T10:21:47.756 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:47 smithi159 ceph-mon[93039]: from='osd.11 [v2:172.21.15.171:6824/596725174,v1:172.21.15.171:6825/596725174]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2023-03-27T10:21:47.756 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:47 smithi159 ceph-mon[93039]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2023-03-27T10:21:47.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:47 smithi161 ceph-mon[94333]: from='osd.11 [v2:172.21.15.171:6824/596725174,v1:172.21.15.171:6825/596725174]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2023-03-27T10:21:47.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:47 smithi161 ceph-mon[94333]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2023-03-27T10:21:48.464 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.464+0000 7f80e607a700 1 -- 172.21.15.159:0/1440241716 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f80e00ffea0 msgr2=0x7f80e01002f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:48.464 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.464+0000 7f80e607a700 1 --2- 172.21.15.159:0/1440241716 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f80e00ffea0 0x7f80e01002f0 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f80d4009990 tx=0x7f80d4009ca0 comp rx=0 tx=0).stop 2023-03-27T10:21:48.465 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.464+0000 7f80e607a700 1 -- 172.21.15.159:0/1440241716 shutdown_connections 2023-03-27T10:21:48.465 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.464+0000 7f80e607a700 1 --2- 172.21.15.159:0/1440241716 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f80e00feca0 0x7f80e00ff0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:48.465 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.464+0000 7f80e607a700 1 --2- 172.21.15.159:0/1440241716 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f80e0100830 0x7f80e01070f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:48.465 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.464+0000 7f80e607a700 1 --2- 172.21.15.159:0/1440241716 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f80e00ffea0 0x7f80e01002f0 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:48.465 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.464+0000 7f80e607a700 1 -- 172.21.15.159:0/1440241716 >> 172.21.15.159:0/1440241716 conn(0x7f80e00fa2b0 msgr2=0x7f80e00fc720 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:21:48.465 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.464+0000 7f80e607a700 1 -- 172.21.15.159:0/1440241716 shutdown_connections 2023-03-27T10:21:48.465 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.464+0000 7f80e607a700 1 -- 172.21.15.159:0/1440241716 wait complete. 2023-03-27T10:21:48.467 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.465+0000 7f80e607a700 1 Processor -- start 2023-03-27T10:21:48.467 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.465+0000 7f80e607a700 1 -- start start 2023-03-27T10:21:48.467 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.465+0000 7f80e607a700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f80e00feca0 0x7f80e0195dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:48.468 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.465+0000 7f80e607a700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f80e00ffea0 0x7f80e0196300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:48.468 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.465+0000 7f80e607a700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f80e0100830 0x7f80e019d380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:48.468 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.465+0000 7f80e607a700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80e019d950 con 0x7f80e00ffea0 2023-03-27T10:21:48.468 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.465+0000 7f80e607a700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80e019da90 con 0x7f80e0100830 2023-03-27T10:21:48.468 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.465+0000 7f80e607a700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80e019dc20 con 0x7f80e00feca0 2023-03-27T10:21:48.468 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80deffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f80e00ffea0 0x7f80e0196300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:48.468 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80deffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f80e00ffea0 0x7f80e0196300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45258/0 (socket says 172.21.15.159:45258) 2023-03-27T10:21:48.469 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80deffd700 1 -- 172.21.15.159:0/4104720916 learned_addr learned my addr 172.21.15.159:0/4104720916 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:21:48.469 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80df7fe700 1 --2- 172.21.15.159:0/4104720916 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f80e00feca0 0x7f80e0195dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:48.469 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80dffff700 1 --2- 172.21.15.159:0/4104720916 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f80e0100830 0x7f80e019d380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:48.469 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80deffd700 1 -- 172.21.15.159:0/4104720916 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f80e00feca0 msgr2=0x7f80e0195dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:48.469 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80deffd700 1 --2- 172.21.15.159:0/4104720916 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f80e00feca0 0x7f80e0195dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:48.469 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80deffd700 1 -- 172.21.15.159:0/4104720916 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f80e0100830 msgr2=0x7f80e019d380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:48.470 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80deffd700 1 --2- 172.21.15.159:0/4104720916 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f80e0100830 0x7f80e019d380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:48.470 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80deffd700 1 -- 172.21.15.159:0/4104720916 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f80d40095f0 con 0x7f80e00ffea0 2023-03-27T10:21:48.470 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80dffff700 1 --2- 172.21.15.159:0/4104720916 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f80e0100830 0x7f80e019d380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-03-27T10:21:48.470 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80deffd700 1 --2- 172.21.15.159:0/4104720916 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f80e00ffea0 0x7f80e0196300 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f80d4005910 tx=0x7f80d400f7b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:21:48.470 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80dcff9700 1 -- 172.21.15.159:0/4104720916 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80d401d070 con 0x7f80e00ffea0 2023-03-27T10:21:48.470 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80e607a700 1 -- 172.21.15.159:0/4104720916 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f80e010a760 con 0x7f80e00ffea0 2023-03-27T10:21:48.470 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80dcff9700 1 -- 172.21.15.159:0/4104720916 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f80d400fc60 con 0x7f80e00ffea0 2023-03-27T10:21:48.470 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80dcff9700 1 -- 172.21.15.159:0/4104720916 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80d40176e0 con 0x7f80e00ffea0 2023-03-27T10:21:48.471 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.466+0000 7f80e607a700 1 -- 172.21.15.159:0/4104720916 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f80e010a5d0 con 0x7f80e00ffea0 2023-03-27T10:21:48.471 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.467+0000 7f80e607a700 1 -- 172.21.15.159:0/4104720916 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f80e019e180 con 0x7f80e00ffea0 2023-03-27T10:21:48.472 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.468+0000 7f80dcff9700 1 -- 172.21.15.159:0/4104720916 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f80d400c040 con 0x7f80e00ffea0 2023-03-27T10:21:48.472 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.469+0000 7f80dcff9700 1 --2- 172.21.15.159:0/4104720916 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f80c809b3a0 0x7f80c809d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:48.472 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.469+0000 7f80df7fe700 1 --2- 172.21.15.159:0/4104720916 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f80c809b3a0 0x7f80c809d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:48.472 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.469+0000 7f80dcff9700 1 -- 172.21.15.159:0/4104720916 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 6847+0+0 (secure 0 0 0) 0x7f80d40c5f90 con 0x7f80e00ffea0 2023-03-27T10:21:48.472 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.469+0000 7f80df7fe700 1 --2- 172.21.15.159:0/4104720916 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f80c809b3a0 0x7f80c809d850 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f80d0006fd0 tx=0x7f80d0008040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:21:48.472 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.472+0000 7f80dcff9700 1 -- 172.21.15.159:0/4104720916 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f80d4091e60 con 0x7f80e00ffea0 2023-03-27T10:21:48.656 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.656+0000 7f80e607a700 1 -- 172.21.15.159:0/4104720916 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd stat", "format": "json"} v 0) v1 -- 0x7f80e019eaa0 con 0x7f80e00ffea0 2023-03-27T10:21:48.656 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.657+0000 7f80dcff9700 1 -- 172.21.15.159:0/4104720916 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd stat", "format": "json"}]=0 v79) v1 ==== 74+0+133 (secure 0 0 0) 0x7f80d40270a0 con 0x7f80e00ffea0 2023-03-27T10:21:48.657 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:21:48.658 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.659+0000 7f80e607a700 1 -- 172.21.15.159:0/4104720916 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f80c809b3a0 msgr2=0x7f80c809d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:48.658 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.659+0000 7f80e607a700 1 --2- 172.21.15.159:0/4104720916 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f80c809b3a0 0x7f80c809d850 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f80d0006fd0 tx=0x7f80d0008040 comp rx=0 tx=0).stop 2023-03-27T10:21:48.658 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.659+0000 7f80e607a700 1 -- 172.21.15.159:0/4104720916 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f80e00ffea0 msgr2=0x7f80e0196300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:48.658 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.659+0000 7f80e607a700 1 --2- 172.21.15.159:0/4104720916 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f80e00ffea0 0x7f80e0196300 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f80d4005910 tx=0x7f80d400f7b0 comp rx=0 tx=0).stop 2023-03-27T10:21:48.659 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.659+0000 7f80e607a700 1 -- 172.21.15.159:0/4104720916 shutdown_connections 2023-03-27T10:21:48.659 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.659+0000 7f80e607a700 1 --2- 172.21.15.159:0/4104720916 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f80c809b3a0 0x7f80c809d850 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:48.659 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.659+0000 7f80e607a700 1 --2- 172.21.15.159:0/4104720916 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f80e00feca0 0x7f80e0195dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:48.659 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.659+0000 7f80e607a700 1 --2- 172.21.15.159:0/4104720916 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f80e0100830 0x7f80e019d380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:48.659 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.659+0000 7f80e607a700 1 --2- 172.21.15.159:0/4104720916 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f80e00ffea0 0x7f80e0196300 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:48.659 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.659+0000 7f80e607a700 1 -- 172.21.15.159:0/4104720916 >> 172.21.15.159:0/4104720916 conn(0x7f80e00fa2b0 msgr2=0x7f80e00680a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:21:48.659 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.659+0000 7f80e607a700 1 -- 172.21.15.159:0/4104720916 shutdown_connections 2023-03-27T10:21:48.660 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:48.659+0000 7f80e607a700 1 -- 172.21.15.159:0/4104720916 wait complete. 2023-03-27T10:21:48.726 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:21:48 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:21:48.444+0000 7f4048b32700 -1 osd.11 0 waiting for initial osdmap 2023-03-27T10:21:48.727 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:21:48 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:21:48.468+0000 7f4041120700 -1 osd.11 79 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-03-27T10:21:48.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:48 smithi171 ceph-mon[94808]: pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail 2023-03-27T10:21:48.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:48 smithi171 ceph-mon[94808]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2023-03-27T10:21:48.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:48 smithi171 ceph-mon[94808]: osdmap e78: 12 total, 11 up, 12 in 2023-03-27T10:21:48.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:48 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:48.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:48 smithi171 ceph-mon[94808]: from='osd.11 [v2:172.21.15.171:6824/596725174,v1:172.21.15.171:6825/596725174]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:21:48.728 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:48 smithi171 ceph-mon[94808]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:21:48.781 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:48 smithi159 ceph-mon[93039]: pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail 2023-03-27T10:21:48.781 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:48 smithi159 ceph-mon[93039]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2023-03-27T10:21:48.781 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:48 smithi159 ceph-mon[93039]: osdmap e78: 12 total, 11 up, 12 in 2023-03-27T10:21:48.781 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:48 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:48.781 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:48 smithi159 ceph-mon[93039]: from='osd.11 [v2:172.21.15.171:6824/596725174,v1:172.21.15.171:6825/596725174]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:21:48.782 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:48 smithi159 ceph-mon[93039]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:21:48.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:48 smithi161 ceph-mon[94333]: pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail 2023-03-27T10:21:48.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:48 smithi161 ceph-mon[94333]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2023-03-27T10:21:48.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:48 smithi161 ceph-mon[94333]: osdmap e78: 12 total, 11 up, 12 in 2023-03-27T10:21:48.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:48 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:48.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:48 smithi161 ceph-mon[94333]: from='osd.11 [v2:172.21.15.171:6824/596725174,v1:172.21.15.171:6825/596725174]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:21:48.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:48 smithi161 ceph-mon[94333]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2023-03-27T10:21:49.071 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":79,"num_osds":12,"num_up_osds":11,"osd_up_since":1679912487,"num_in_osds":12,"osd_in_since":1679912492,"num_remapped_pgs":0} 2023-03-27T10:21:49.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:49 smithi171 ceph-mon[94808]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2023-03-27T10:21:49.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:49 smithi171 ceph-mon[94808]: osdmap e79: 12 total, 11 up, 12 in 2023-03-27T10:21:49.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:49.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:49.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:49 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/4104720916' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-03-27T10:21:49.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:49.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:49.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:49.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:49.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:49.728 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:49.728 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:49.728 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:49.728 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:49.728 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:49 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:49.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:49 smithi161 ceph-mon[94333]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2023-03-27T10:21:49.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:49 smithi161 ceph-mon[94333]: osdmap e79: 12 total, 11 up, 12 in 2023-03-27T10:21:49.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:49.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:49.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:49 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/4104720916' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-03-27T10:21:49.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:49.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:49.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:49.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:49.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:49.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:49.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:49.836 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:49.836 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:49.836 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:49 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:49.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:49 smithi159 ceph-mon[93039]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2023-03-27T10:21:49.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:49 smithi159 ceph-mon[93039]: osdmap e79: 12 total, 11 up, 12 in 2023-03-27T10:21:49.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:49.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:49.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:49 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/4104720916' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-03-27T10:21:49.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:49.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:49.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:49.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:49.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:49.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2023-03-27T10:21:49.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:49.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:21:49.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:21:49.867 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:49 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:21:50.073 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph osd stat -f json 2023-03-27T10:21:50.724 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:21:50.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:50 smithi171 ceph-mon[94808]: purged_snaps scrub starts 2023-03-27T10:21:50.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:50 smithi171 ceph-mon[94808]: purged_snaps scrub ok 2023-03-27T10:21:50.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:50 smithi171 ceph-mon[94808]: Detected new or changed devices on smithi171 2023-03-27T10:21:50.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:50 smithi171 ceph-mon[94808]: Adjusting osd_memory_target on smithi171 to 4292M 2023-03-27T10:21:50.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:50 smithi171 ceph-mon[94808]: pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail 2023-03-27T10:21:50.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:50 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:50.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:50 smithi171 ceph-mon[94808]: osd.11 [v2:172.21.15.171:6824/596725174,v1:172.21.15.171:6825/596725174] boot 2023-03-27T10:21:50.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:50 smithi171 ceph-mon[94808]: osdmap e80: 12 total, 12 up, 12 in 2023-03-27T10:21:50.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:50 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:50.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:50 smithi161 ceph-mon[94333]: purged_snaps scrub starts 2023-03-27T10:21:50.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:50 smithi161 ceph-mon[94333]: purged_snaps scrub ok 2023-03-27T10:21:50.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:50 smithi161 ceph-mon[94333]: Detected new or changed devices on smithi171 2023-03-27T10:21:50.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:50 smithi161 ceph-mon[94333]: Adjusting osd_memory_target on smithi171 to 4292M 2023-03-27T10:21:50.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:50 smithi161 ceph-mon[94333]: pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail 2023-03-27T10:21:50.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:50 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:50.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:50 smithi161 ceph-mon[94333]: osd.11 [v2:172.21.15.171:6824/596725174,v1:172.21.15.171:6825/596725174] boot 2023-03-27T10:21:50.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:50 smithi161 ceph-mon[94333]: osdmap e80: 12 total, 12 up, 12 in 2023-03-27T10:21:50.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:50 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:50 smithi159 ceph-mon[93039]: purged_snaps scrub starts 2023-03-27T10:21:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:50 smithi159 ceph-mon[93039]: purged_snaps scrub ok 2023-03-27T10:21:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:50 smithi159 ceph-mon[93039]: Detected new or changed devices on smithi171 2023-03-27T10:21:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:50 smithi159 ceph-mon[93039]: Adjusting osd_memory_target on smithi171 to 4292M 2023-03-27T10:21:50.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:50 smithi159 ceph-mon[93039]: pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 67 MiB used, 983 GiB / 983 GiB avail 2023-03-27T10:21:50.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:50 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:50.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:50 smithi159 ceph-mon[93039]: osd.11 [v2:172.21.15.171:6824/596725174,v1:172.21.15.171:6825/596725174] boot 2023-03-27T10:21:50.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:50 smithi159 ceph-mon[93039]: osdmap e80: 12 total, 12 up, 12 in 2023-03-27T10:21:50.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:50 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-03-27T10:21:51.895 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.896+0000 7f6b090f5700 1 -- 172.21.15.159:0/561914434 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6b040feca0 msgr2=0x7f6b040ff0b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:51.896 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.896+0000 7f6b090f5700 1 --2- 172.21.15.159:0/561914434 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6b040feca0 0x7f6b040ff0b0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f6af4009990 tx=0x7f6af4009ca0 comp rx=0 tx=0).stop 2023-03-27T10:21:51.896 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.896+0000 7f6b090f5700 1 -- 172.21.15.159:0/561914434 shutdown_connections 2023-03-27T10:21:51.896 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.896+0000 7f6b090f5700 1 --2- 172.21.15.159:0/561914434 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f6b04100830 0x7f6b041070f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:51.896 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.896+0000 7f6b090f5700 1 --2- 172.21.15.159:0/561914434 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6b040feca0 0x7f6b040ff0b0 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:51.896 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.896+0000 7f6b090f5700 1 --2- 172.21.15.159:0/561914434 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6b040ffea0 0x7f6b041002f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:51.897 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.896+0000 7f6b090f5700 1 -- 172.21.15.159:0/561914434 >> 172.21.15.159:0/561914434 conn(0x7f6b040fa2b0 msgr2=0x7f6b040fc720 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:21:51.897 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.896+0000 7f6b090f5700 1 -- 172.21.15.159:0/561914434 shutdown_connections 2023-03-27T10:21:51.897 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.896+0000 7f6b090f5700 1 -- 172.21.15.159:0/561914434 wait complete. 2023-03-27T10:21:51.898 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.897+0000 7f6b090f5700 1 Processor -- start 2023-03-27T10:21:51.898 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.897+0000 7f6b090f5700 1 -- start start 2023-03-27T10:21:51.898 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.897+0000 7f6b090f5700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f6b040feca0 0x7f6b04068c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:51.898 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.897+0000 7f6b090f5700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6b040ffea0 0x7f6b0419a7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:51.898 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.897+0000 7f6b090f5700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6b04100830 0x7f6b0419cdf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:51.899 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.897+0000 7f6b090f5700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b0419d590 con 0x7f6b04100830 2023-03-27T10:21:51.899 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.897+0000 7f6b090f5700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b0419d700 con 0x7f6b040feca0 2023-03-27T10:21:51.899 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.897+0000 7f6b090f5700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b0419d8c0 con 0x7f6b040ffea0 2023-03-27T10:21:51.899 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.897+0000 7f6b0359e700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6b04100830 0x7f6b0419cdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:51.899 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.897+0000 7f6b0359e700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6b04100830 0x7f6b0419cdf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45278/0 (socket says 172.21.15.159:45278) 2023-03-27T10:21:51.899 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.897+0000 7f6b0359e700 1 -- 172.21.15.159:0/3563475263 learned_addr learned my addr 172.21.15.159:0/3563475263 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:21:51.899 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.897+0000 7f6b02d9d700 1 --2- 172.21.15.159:0/3563475263 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f6b040feca0 0x7f6b04068c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:51.900 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.897+0000 7f6b0259c700 1 --2- 172.21.15.159:0/3563475263 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6b040ffea0 0x7f6b0419a7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:51.900 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.898+0000 7f6b0359e700 1 -- 172.21.15.159:0/3563475263 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6b040ffea0 msgr2=0x7f6b0419a7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:51.900 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.898+0000 7f6b0359e700 1 --2- 172.21.15.159:0/3563475263 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6b040ffea0 0x7f6b0419a7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:51.900 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.898+0000 7f6b0359e700 1 -- 172.21.15.159:0/3563475263 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f6b040feca0 msgr2=0x7f6b04068c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:51.900 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.898+0000 7f6b0359e700 1 --2- 172.21.15.159:0/3563475263 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f6b040feca0 0x7f6b04068c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:51.900 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.898+0000 7f6b0359e700 1 -- 172.21.15.159:0/3563475263 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6af40095f0 con 0x7f6b04100830 2023-03-27T10:21:51.900 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.898+0000 7f6b0359e700 1 --2- 172.21.15.159:0/3563475263 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6b04100830 0x7f6b0419cdf0 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f6afc007ee0 tx=0x7f6afc00d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:21:51.900 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.898+0000 7f6afbfff700 1 -- 172.21.15.159:0/3563475263 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6afc028070 con 0x7f6b04100830 2023-03-27T10:21:51.901 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.898+0000 7f6afbfff700 1 -- 172.21.15.159:0/3563475263 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6afc009390 con 0x7f6b04100830 2023-03-27T10:21:51.901 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.898+0000 7f6b090f5700 1 -- 172.21.15.159:0/3563475263 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6b0410a5f0 con 0x7f6b04100830 2023-03-27T10:21:51.901 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.898+0000 7f6afbfff700 1 -- 172.21.15.159:0/3563475263 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6afc005720 con 0x7f6b04100830 2023-03-27T10:21:51.901 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.898+0000 7f6b090f5700 1 -- 172.21.15.159:0/3563475263 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6b0410a460 con 0x7f6b04100830 2023-03-27T10:21:51.902 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.899+0000 7f6b090f5700 1 -- 172.21.15.159:0/3563475263 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6b04105b90 con 0x7f6b04100830 2023-03-27T10:21:51.902 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.900+0000 7f6afbfff700 1 -- 172.21.15.159:0/3563475263 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f6afc022030 con 0x7f6b04100830 2023-03-27T10:21:51.902 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.901+0000 7f6afbfff700 1 --2- 172.21.15.159:0/3563475263 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6af009b3a0 0x7f6af009d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:51.902 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.901+0000 7f6b02d9d700 1 --2- 172.21.15.159:0/3563475263 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6af009b3a0 0x7f6af009d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:51.902 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.901+0000 7f6afbfff700 1 -- 172.21.15.159:0/3563475263 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f6afc0d5380 con 0x7f6b04100830 2023-03-27T10:21:51.903 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.901+0000 7f6b02d9d700 1 --2- 172.21.15.159:0/3563475263 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6af009b3a0 0x7f6af009d850 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f6af400b5c0 tx=0x7f6af4005fd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:21:51.904 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:51.905+0000 7f6afbfff700 1 -- 172.21.15.159:0/3563475263 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f6afc0a10a0 con 0x7f6b04100830 2023-03-27T10:21:52.089 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:52.089+0000 7f6b090f5700 1 -- 172.21.15.159:0/3563475263 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd stat", "format": "json"} v 0) v1 -- 0x7f6b0402cdb0 con 0x7f6b04100830 2023-03-27T10:21:52.089 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:52.090+0000 7f6afbfff700 1 -- 172.21.15.159:0/3563475263 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd stat", "format": "json"}]=0 v81) v1 ==== 74+0+133 (secure 0 0 0) 0x7f6afc0a46c0 con 0x7f6b04100830 2023-03-27T10:21:52.090 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:21:52.091 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:52.092+0000 7f6b090f5700 1 -- 172.21.15.159:0/3563475263 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6af009b3a0 msgr2=0x7f6af009d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:52.091 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:52.092+0000 7f6b090f5700 1 --2- 172.21.15.159:0/3563475263 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6af009b3a0 0x7f6af009d850 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f6af400b5c0 tx=0x7f6af4005fd0 comp rx=0 tx=0).stop 2023-03-27T10:21:52.092 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:52.092+0000 7f6b090f5700 1 -- 172.21.15.159:0/3563475263 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6b04100830 msgr2=0x7f6b0419cdf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:52.092 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:52.092+0000 7f6b090f5700 1 --2- 172.21.15.159:0/3563475263 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6b04100830 0x7f6b0419cdf0 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f6afc007ee0 tx=0x7f6afc00d3b0 comp rx=0 tx=0).stop 2023-03-27T10:21:52.092 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:52.092+0000 7f6b090f5700 1 -- 172.21.15.159:0/3563475263 shutdown_connections 2023-03-27T10:21:52.092 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:52.092+0000 7f6b090f5700 1 --2- 172.21.15.159:0/3563475263 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f6af009b3a0 0x7f6af009d850 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:52.092 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:52.092+0000 7f6b090f5700 1 --2- 172.21.15.159:0/3563475263 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f6b04100830 0x7f6b0419cdf0 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:52.092 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:52.092+0000 7f6b090f5700 1 --2- 172.21.15.159:0/3563475263 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f6b040feca0 0x7f6b04068c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:52.092 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:52.092+0000 7f6b090f5700 1 --2- 172.21.15.159:0/3563475263 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f6b040ffea0 0x7f6b0419a7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:52.093 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:52.092+0000 7f6b090f5700 1 -- 172.21.15.159:0/3563475263 >> 172.21.15.159:0/3563475263 conn(0x7f6b040fa2b0 msgr2=0x7f6b040fc4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:21:52.093 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:52.092+0000 7f6b090f5700 1 -- 172.21.15.159:0/3563475263 shutdown_connections 2023-03-27T10:21:52.093 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:52.092+0000 7f6b090f5700 1 -- 172.21.15.159:0/3563475263 wait complete. 2023-03-27T10:21:52.196 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:51 smithi159 ceph-mon[93039]: osdmap e81: 12 total, 12 up, 12 in 2023-03-27T10:21:52.197 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:51 smithi159 ceph-mon[93039]: pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 73 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:21:52.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:51 smithi171 ceph-mon[94808]: osdmap e81: 12 total, 12 up, 12 in 2023-03-27T10:21:52.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:51 smithi171 ceph-mon[94808]: pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 73 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:21:52.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:51 smithi161 ceph-mon[94333]: osdmap e81: 12 total, 12 up, 12 in 2023-03-27T10:21:52.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:51 smithi161 ceph-mon[94333]: pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 73 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:21:52.485 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":81,"num_osds":12,"num_up_osds":12,"osd_up_since":1679912509,"num_in_osds":12,"osd_in_since":1679912492,"num_remapped_pgs":0} 2023-03-27T10:21:52.486 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph osd dump --format=json 2023-03-27T10:21:53.128 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:21:53.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:52 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3563475263' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-03-27T10:21:53.327 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:52 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3563475263' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-03-27T10:21:53.333 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:52 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3563475263' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-03-27T10:21:54.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:53 smithi171 ceph-mon[94808]: pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 73 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:21:54.299 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.299+0000 7f91a166f700 1 -- 172.21.15.159:0/985663371 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f919c0ff900 msgr2=0x7f919c1038d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:54.299 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.299+0000 7f91a166f700 1 --2- 172.21.15.159:0/985663371 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f919c0ff900 0x7f919c1038d0 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f918c0099c0 tx=0x7f918c009cd0 comp rx=0 tx=0).stop 2023-03-27T10:21:54.299 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.299+0000 7f91a166f700 1 -- 172.21.15.159:0/985663371 shutdown_connections 2023-03-27T10:21:54.299 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.299+0000 7f91a166f700 1 --2- 172.21.15.159:0/985663371 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f919c104000 0x7f919c10bb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:54.300 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.299+0000 7f91a166f700 1 --2- 172.21.15.159:0/985663371 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f919c0fef60 0x7f919c0ff330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:54.300 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.299+0000 7f91a166f700 1 --2- 172.21.15.159:0/985663371 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f919c0ff900 0x7f919c1038d0 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:54.300 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.299+0000 7f91a166f700 1 -- 172.21.15.159:0/985663371 >> 172.21.15.159:0/985663371 conn(0x7f919c075e70 msgr2=0x7f919c076270 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:21:54.300 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.299+0000 7f91a166f700 1 -- 172.21.15.159:0/985663371 shutdown_connections 2023-03-27T10:21:54.300 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.299+0000 7f91a166f700 1 -- 172.21.15.159:0/985663371 wait complete. 2023-03-27T10:21:54.302 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.300+0000 7f91a166f700 1 Processor -- start 2023-03-27T10:21:54.302 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.300+0000 7f91a166f700 1 -- start start 2023-03-27T10:21:54.302 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.300+0000 7f91a166f700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f919c0fef60 0x7f919c196da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:54.302 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.300+0000 7f91a166f700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f919c0ff900 0x7f919c1972e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:54.303 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.300+0000 7f91a166f700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f919c104000 0x7f919c1a1e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:54.303 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.300+0000 7f91a166f700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f919c197bf0 con 0x7f919c104000 2023-03-27T10:21:54.303 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.300+0000 7f91a166f700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f919c1a2360 con 0x7f919c0ff900 2023-03-27T10:21:54.303 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.300+0000 7f91a166f700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f919c1a24d0 con 0x7f919c0fef60 2023-03-27T10:21:54.303 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.300+0000 7f919b7fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f919c104000 0x7f919c1a1e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:54.303 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.300+0000 7f919b7fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f919c104000 0x7f919c1a1e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45290/0 (socket says 172.21.15.159:45290) 2023-03-27T10:21:54.303 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.300+0000 7f919b7fe700 1 -- 172.21.15.159:0/472123574 learned_addr learned my addr 172.21.15.159:0/472123574 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:21:54.303 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.300+0000 7f919affd700 1 --2- 172.21.15.159:0/472123574 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f919c0fef60 0x7f919c196da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:54.304 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.300+0000 7f919a7fc700 1 --2- 172.21.15.159:0/472123574 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f919c0ff900 0x7f919c1972e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:54.304 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.301+0000 7f919b7fe700 1 -- 172.21.15.159:0/472123574 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f919c0fef60 msgr2=0x7f919c196da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:54.304 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.301+0000 7f919b7fe700 1 --2- 172.21.15.159:0/472123574 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f919c0fef60 0x7f919c196da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:54.304 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.301+0000 7f919b7fe700 1 -- 172.21.15.159:0/472123574 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f919c0ff900 msgr2=0x7f919c1972e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:54.304 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.301+0000 7f919b7fe700 1 --2- 172.21.15.159:0/472123574 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f919c0ff900 0x7f919c1972e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:54.304 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.301+0000 7f919b7fe700 1 -- 172.21.15.159:0/472123574 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f918c0095f0 con 0x7f919c104000 2023-03-27T10:21:54.304 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.301+0000 7f919a7fc700 1 --2- 172.21.15.159:0/472123574 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f919c0ff900 0x7f919c1972e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:21:54.305 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.301+0000 7f919b7fe700 1 --2- 172.21.15.159:0/472123574 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f919c104000 0x7f919c1a1e20 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f919400b970 tx=0x7f919400bd30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:21:54.305 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.301+0000 7f9183fff700 1 -- 172.21.15.159:0/472123574 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f919400c9b0 con 0x7f919c104000 2023-03-27T10:21:54.305 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.301+0000 7f91a166f700 1 -- 172.21.15.159:0/472123574 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f919c10edd0 con 0x7f919c104000 2023-03-27T10:21:54.305 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.301+0000 7f9183fff700 1 -- 172.21.15.159:0/472123574 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9194014440 con 0x7f919c104000 2023-03-27T10:21:54.305 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.301+0000 7f9183fff700 1 -- 172.21.15.159:0/472123574 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91940075d0 con 0x7f919c104000 2023-03-27T10:21:54.305 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.301+0000 7f91a166f700 1 -- 172.21.15.159:0/472123574 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f919c10ec40 con 0x7f919c104000 2023-03-27T10:21:54.305 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.302+0000 7f91a166f700 1 -- 172.21.15.159:0/472123574 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f919c10ef60 con 0x7f919c104000 2023-03-27T10:21:54.305 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.303+0000 7f9183fff700 1 -- 172.21.15.159:0/472123574 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f919401b030 con 0x7f919c104000 2023-03-27T10:21:54.306 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.303+0000 7f9183fff700 1 --2- 172.21.15.159:0/472123574 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f918409b420 0x7f918409d8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:54.306 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.303+0000 7f919affd700 1 --2- 172.21.15.159:0/472123574 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f918409b420 0x7f918409d8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:54.306 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.304+0000 7f9183fff700 1 -- 172.21.15.159:0/472123574 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f9194006e40 con 0x7f919c104000 2023-03-27T10:21:54.307 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.304+0000 7f919affd700 1 --2- 172.21.15.159:0/472123574 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f918409b420 0x7f918409d8d0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f9190005950 tx=0x7f91900058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:21:54.307 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.307+0000 7f9183fff700 1 -- 172.21.15.159:0/472123574 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f919404d960 con 0x7f919c104000 2023-03-27T10:21:54.333 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:53 smithi161 ceph-mon[94333]: pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 73 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:21:54.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:53 smithi159 ceph-mon[93039]: pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 73 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:21:54.490 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.490+0000 7f91a166f700 1 -- 172.21.15.159:0/472123574 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f919c066990 con 0x7f919c104000 2023-03-27T10:21:54.491 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.491+0000 7f9183fff700 1 -- 172.21.15.159:0/472123574 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v81) v1 ==== 74+0+18455 (secure 0 0 0) 0x7f9194019090 con 0x7f919c104000 2023-03-27T10:21:54.491 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:21:54.492 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":81,"fsid":"2c3cd140-cc88-11ed-9afe-001a4aab830c","created":"2023-03-27T10:15:32.617230+0000","modified":"2023-03-27T10:21:50.939009+0000","last_up_change":"2023-03-27T10:21:49.442480+0000","last_in_change":"2023-03-27T10:21:32.265278+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-03-27T10:18:35.277335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f47f99b9-47f9-4d94-83d9-33b1278be17e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6803","nonce":87273908}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6805","nonce":87273908}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6809","nonce":87273908}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6807","nonce":87273908}]},"public_addr":"172.21.15.159:6803/87273908","cluster_addr":"172.21.15.159:6805/87273908","heartbeat_back_addr":"172.21.15.159:6809/87273908","heartbeat_front_addr":"172.21.15.159:6807/87273908","state":["exists","up"]},{"osd":1,"uuid":"97f66da9-22b4-4eb5-9c77-9d37cf0fca82","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6800","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6801","nonce":2094166586}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6802","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6803","nonce":2094166586}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6806","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6807","nonce":2094166586}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6804","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6805","nonce":2094166586}]},"public_addr":"172.21.15.161:6801/2094166586","cluster_addr":"172.21.15.161:6803/2094166586","heartbeat_back_addr":"172.21.15.161:6807/2094166586","heartbeat_front_addr":"172.21.15.161:6805/2094166586","state":["exists","up"]},{"osd":2,"uuid":"ba9f5f82-3aef-42f7-a9e6-0e07aa7d8df1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6801","nonce":2724827688}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6803","nonce":2724827688}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6807","nonce":2724827688}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6805","nonce":2724827688}]},"public_addr":"172.21.15.171:6801/2724827688","cluster_addr":"172.21.15.171:6803/2724827688","heartbeat_back_addr":"172.21.15.171:6807/2724827688","heartbeat_front_addr":"172.21.15.171:6805/2724827688","state":["exists","up"]},{"osd":3,"uuid":"4ef4b519-fdef-4d38-b387-bc14ea381bd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6811","nonce":1992276871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6813","nonce":1992276871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6817","nonce":1992276871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6815","nonce":1992276871}]},"public_addr":"172.21.15.159:6811/1992276871","cluster_addr":"172.21.15.159:6813/1992276871","heartbeat_back_addr":"172.21.15.159:6817/1992276871","heartbeat_front_addr":"172.21.15.159:6815/1992276871","state":["exists","up"]},{"osd":4,"uuid":"982675c1-dd87-46b4-97ef-a5dde1b1d777","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6808","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6809","nonce":3978602555}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6810","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6811","nonce":3978602555}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6814","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6815","nonce":3978602555}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6812","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6813","nonce":3978602555}]},"public_addr":"172.21.15.161:6809/3978602555","cluster_addr":"172.21.15.161:6811/3978602555","heartbeat_back_addr":"172.21.15.161:6815/3978602555","heartbeat_front_addr":"172.21.15.161:6813/3978602555","state":["exists","up"]},{"osd":5,"uuid":"b4de4a85-4f91-4194-a1ce-db12f1898c4f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6809","nonce":569611891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6811","nonce":569611891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6815","nonce":569611891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6813","nonce":569611891}]},"public_addr":"172.21.15.171:6809/569611891","cluster_addr":"172.21.15.171:6811/569611891","heartbeat_back_addr":"172.21.15.171:6815/569611891","heartbeat_front_addr":"172.21.15.171:6813/569611891","state":["exists","up"]},{"osd":6,"uuid":"dff7d5f4-655f-4baa-9084-03591a4b3bb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6819","nonce":1176978350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6821","nonce":1176978350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6825","nonce":1176978350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6823","nonce":1176978350}]},"public_addr":"172.21.15.159:6819/1176978350","cluster_addr":"172.21.15.159:6821/1176978350","heartbeat_back_addr":"172.21.15.159:6825/1176978350","heartbeat_front_addr":"172.21.15.159:6823/1176978350","state":["exists","up"]},{"osd":7,"uuid":"30f026d9-6963-4468-95e8-78c7ed2e89f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6816","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6817","nonce":2098174471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6818","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6819","nonce":2098174471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6822","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6823","nonce":2098174471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6820","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6821","nonce":2098174471}]},"public_addr":"172.21.15.161:6817/2098174471","cluster_addr":"172.21.15.161:6819/2098174471","heartbeat_back_addr":"172.21.15.161:6823/2098174471","heartbeat_front_addr":"172.21.15.161:6821/2098174471","state":["exists","up"]},{"osd":8,"uuid":"9f69fbcc-32ff-4d4b-b562-ee0a12745cc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6817","nonce":1033586906}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6819","nonce":1033586906}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6823","nonce":1033586906}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6821","nonce":1033586906}]},"public_addr":"172.21.15.171:6817/1033586906","cluster_addr":"172.21.15.171:6819/1033586906","heartbeat_back_addr":"172.21.15.171:6823/1033586906","heartbeat_front_addr":"172.21.15.171:6821/1033586906","state":["exists","up"]},{"osd":9,"uuid":"d7fb9eed-901b-4943-9529-6edcba2385e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6827","nonce":3285179829}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6829","nonce":3285179829}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6832","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6833","nonce":3285179829}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6831","nonce":3285179829}]},"public_addr":"172.21.15.159:6827/3285179829","cluster_addr":"172.21.15.159:6829/3285179829","heartbeat_back_addr":"172.21.15.159:6833/3285179829","heartbeat_front_addr":"172.21.15.159:6831/3285179829","state":["exists","up"]},{"osd":10,"uuid":"4871ca03-30ae-4e87-942c-9441dd31fb40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6824","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6825","nonce":946677932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6826","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6827","nonce":946677932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6830","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6831","nonce":946677932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6828","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6829","nonce":946677932}]},"public_addr":"172.21.15.161:6825/946677932","cluster_addr":"172.21.15.161:6827/946677932","heartbeat_back_addr":"172.21.15.161:6831/946677932","heartbeat_front_addr":"172.21.15.161:6829/946677932","state":["exists","up"]},{"osd":11,"uuid":"1fb37d9a-ed52-4c0d-a082-65e17d242106","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6825","nonce":596725174}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6827","nonce":596725174}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6831","nonce":596725174}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6829","nonce":596725174}]},"public_addr":"172.21.15.171:6825/596725174","cluster_addr":"172.21.15.171:6827/596725174","heartbeat_back_addr":"172.21.15.171:6831/596725174","heartbeat_front_addr":"172.21.15.171:6829/596725174","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:17:49.128592+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:10.944593+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:32.989432+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:54.675294+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:16.543485+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:38.390326+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:59.566202+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:20:21.348420+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:20:43.383576+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:04.710992+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:26.286573+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:48.277317+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.159:0/653469093":"2023-03-28T10:16:26.900382+0000","172.21.15.159:6800/3561797228":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/3045586257":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/2859804517":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6801/3561797228":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/3487813906":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/1306320900":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6800/1729835725":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6801/1729835725":"2023-03-28T10:16:04.737100+0000","172.21.15.159:0/72206061":"2023-03-28T10:16:04.737100+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-03-27T10:21:54.493 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.494+0000 7f91a166f700 1 -- 172.21.15.159:0/472123574 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f918409b420 msgr2=0x7f918409d8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:54.493 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.494+0000 7f91a166f700 1 --2- 172.21.15.159:0/472123574 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f918409b420 0x7f918409d8d0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f9190005950 tx=0x7f91900058e0 comp rx=0 tx=0).stop 2023-03-27T10:21:54.493 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.494+0000 7f91a166f700 1 -- 172.21.15.159:0/472123574 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f919c104000 msgr2=0x7f919c1a1e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:54.494 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.494+0000 7f91a166f700 1 --2- 172.21.15.159:0/472123574 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f919c104000 0x7f919c1a1e20 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f919400b970 tx=0x7f919400bd30 comp rx=0 tx=0).stop 2023-03-27T10:21:54.494 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.494+0000 7f91a166f700 1 -- 172.21.15.159:0/472123574 shutdown_connections 2023-03-27T10:21:54.494 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.494+0000 7f91a166f700 1 --2- 172.21.15.159:0/472123574 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f918409b420 0x7f918409d8d0 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:54.494 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.494+0000 7f91a166f700 1 --2- 172.21.15.159:0/472123574 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f919c0fef60 0x7f919c196da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:54.494 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.494+0000 7f91a166f700 1 --2- 172.21.15.159:0/472123574 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f919c104000 0x7f919c1a1e20 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:54.494 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.494+0000 7f91a166f700 1 --2- 172.21.15.159:0/472123574 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f919c0ff900 0x7f919c1972e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:54.495 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.494+0000 7f91a166f700 1 -- 172.21.15.159:0/472123574 >> 172.21.15.159:0/472123574 conn(0x7f919c075e70 msgr2=0x7f919c0fc5d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:21:54.495 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.494+0000 7f91a166f700 1 -- 172.21.15.159:0/472123574 shutdown_connections 2023-03-27T10:21:54.495 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:54.494+0000 7f91a166f700 1 -- 172.21.15.159:0/472123574 wait complete. 2023-03-27T10:21:54.868 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-03-27T10:18:35.277335+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2023-03-27T10:21:54.868 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph osd pool get .mgr pg_num 2023-03-27T10:21:55.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:54 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/472123574' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:21:55.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:54 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/472123574' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:21:55.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:54 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/472123574' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:21:55.510 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:21:56.544 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:56 smithi159 ceph-mon[93039]: pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:21:56.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:56 smithi171 ceph-mon[94808]: pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:21:56.752 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.752+0000 7f7f69830700 1 -- 172.21.15.159:0/2022672924 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7f64103fd0 msgr2=0x7f7f6410bb50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:56.752 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.752+0000 7f7f69830700 1 --2- 172.21.15.159:0/2022672924 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7f64103fd0 0x7f7f6410bb50 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f7f5c00b0d0 tx=0x7f7f5c00b3e0 comp rx=0 tx=0).stop 2023-03-27T10:21:56.752 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.752+0000 7f7f69830700 1 -- 172.21.15.159:0/2022672924 shutdown_connections 2023-03-27T10:21:56.752 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.752+0000 7f7f69830700 1 --2- 172.21.15.159:0/2022672924 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7f640fef30 0x7f7f640ff300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:56.752 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.752+0000 7f7f69830700 1 --2- 172.21.15.159:0/2022672924 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7f64103fd0 0x7f7f6410bb50 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:56.752 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.752+0000 7f7f69830700 1 --2- 172.21.15.159:0/2022672924 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7f640ff8d0 0x7f7f641038a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:56.753 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.752+0000 7f7f69830700 1 -- 172.21.15.159:0/2022672924 >> 172.21.15.159:0/2022672924 conn(0x7f7f640fa7f0 msgr2=0x7f7f640fcc00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:21:56.753 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.752+0000 7f7f69830700 1 -- 172.21.15.159:0/2022672924 shutdown_connections 2023-03-27T10:21:56.753 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.752+0000 7f7f69830700 1 -- 172.21.15.159:0/2022672924 wait complete. 2023-03-27T10:21:56.754 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.753+0000 7f7f69830700 1 Processor -- start 2023-03-27T10:21:56.755 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.753+0000 7f7f69830700 1 -- start start 2023-03-27T10:21:56.755 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.753+0000 7f7f69830700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7f640fef30 0x7f7f64195ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:56.755 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.753+0000 7f7f69830700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7f640ff8d0 0x7f7f64196430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:56.755 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.753+0000 7f7f69830700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7f64103fd0 0x7f7f6419a8a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:56.755 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.753+0000 7f7f69830700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7f6419aeb0 con 0x7f7f640ff8d0 2023-03-27T10:21:56.755 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.753+0000 7f7f69830700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7f6419b020 con 0x7f7f64103fd0 2023-03-27T10:21:56.756 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.753+0000 7f7f69830700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7f6419b1e0 con 0x7f7f640fef30 2023-03-27T10:21:56.756 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.753+0000 7f7f627fc700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7f640ff8d0 0x7f7f64196430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:56.756 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.753+0000 7f7f627fc700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7f640ff8d0 0x7f7f64196430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45300/0 (socket says 172.21.15.159:45300) 2023-03-27T10:21:56.756 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.753+0000 7f7f627fc700 1 -- 172.21.15.159:0/1885784286 learned_addr learned my addr 172.21.15.159:0/1885784286 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:21:56.756 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.754+0000 7f7f637fe700 1 --2- 172.21.15.159:0/1885784286 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7f64103fd0 0x7f7f6419a8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:56.756 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.754+0000 7f7f62ffd700 1 --2- 172.21.15.159:0/1885784286 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7f640fef30 0x7f7f64195ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:56.756 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.754+0000 7f7f627fc700 1 -- 172.21.15.159:0/1885784286 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7f640fef30 msgr2=0x7f7f64195ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:56.757 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.754+0000 7f7f627fc700 1 --2- 172.21.15.159:0/1885784286 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7f640fef30 0x7f7f64195ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:56.757 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.754+0000 7f7f627fc700 1 -- 172.21.15.159:0/1885784286 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7f64103fd0 msgr2=0x7f7f6419a8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:56.757 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.754+0000 7f7f627fc700 1 --2- 172.21.15.159:0/1885784286 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7f64103fd0 0x7f7f6419a8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:56.757 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.754+0000 7f7f627fc700 1 -- 172.21.15.159:0/1885784286 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7f5c009d00 con 0x7f7f640ff8d0 2023-03-27T10:21:56.757 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.754+0000 7f7f62ffd700 1 --2- 172.21.15.159:0/1885784286 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7f640fef30 0x7f7f64195ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-03-27T10:21:56.757 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.754+0000 7f7f637fe700 1 --2- 172.21.15.159:0/1885784286 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7f64103fd0 0x7f7f6419a8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:21:56.757 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.754+0000 7f7f627fc700 1 --2- 172.21.15.159:0/1885784286 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7f640ff8d0 0x7f7f64196430 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f7f5400c9e0 tx=0x7f7f5400cda0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:21:56.758 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.754+0000 7f7f6882e700 1 -- 172.21.15.159:0/1885784286 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f54007aa0 con 0x7f7f640ff8d0 2023-03-27T10:21:56.758 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.754+0000 7f7f6882e700 1 -- 172.21.15.159:0/1885784286 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7f54007c00 con 0x7f7f640ff8d0 2023-03-27T10:21:56.758 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.754+0000 7f7f69830700 1 -- 172.21.15.159:0/1885784286 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7f6410ed00 con 0x7f7f640ff8d0 2023-03-27T10:21:56.758 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.754+0000 7f7f6882e700 1 -- 172.21.15.159:0/1885784286 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7f54018660 con 0x7f7f640ff8d0 2023-03-27T10:21:56.758 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.754+0000 7f7f69830700 1 -- 172.21.15.159:0/1885784286 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7f6410eb70 con 0x7f7f640ff8d0 2023-03-27T10:21:56.758 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.755+0000 7f7f69830700 1 -- 172.21.15.159:0/1885784286 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7f640683c0 con 0x7f7f640ff8d0 2023-03-27T10:21:56.759 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.756+0000 7f7f6882e700 1 -- 172.21.15.159:0/1885784286 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f7f5400f450 con 0x7f7f640ff8d0 2023-03-27T10:21:56.759 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.757+0000 7f7f6882e700 1 --2- 172.21.15.159:0/1885784286 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7f4c09b420 0x7f7f4c09d8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:56.760 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.757+0000 7f7f6882e700 1 -- 172.21.15.159:0/1885784286 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f7f540c4c00 con 0x7f7f640ff8d0 2023-03-27T10:21:56.760 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.757+0000 7f7f62ffd700 1 --2- 172.21.15.159:0/1885784286 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7f4c09b420 0x7f7f4c09d8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:56.760 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.757+0000 7f7f62ffd700 1 --2- 172.21.15.159:0/1885784286 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7f4c09b420 0x7f7f4c09d8d0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f7f58005fd0 tx=0x7f7f58005dc0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:21:56.760 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.760+0000 7f7f6882e700 1 -- 172.21.15.159:0/1885784286 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f7f54090a90 con 0x7f7f640ff8d0 2023-03-27T10:21:56.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:56 smithi161 ceph-mon[94333]: pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:21:56.942 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.943+0000 7f7f69830700 1 -- 172.21.15.159:0/1885784286 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f7f641a3030 con 0x7f7f640ff8d0 2023-03-27T10:21:56.943 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.943+0000 7f7f6882e700 1 -- 172.21.15.159:0/1885784286 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v81) v1 ==== 93+0+10 (secure 0 0 0) 0x7f7f540940b0 con 0x7f7f640ff8d0 2023-03-27T10:21:56.943 INFO:teuthology.orchestra.run.smithi159.stdout:pg_num: 1 2023-03-27T10:21:56.945 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.946+0000 7f7f69830700 1 -- 172.21.15.159:0/1885784286 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7f4c09b420 msgr2=0x7f7f4c09d8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:56.945 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.946+0000 7f7f69830700 1 --2- 172.21.15.159:0/1885784286 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7f4c09b420 0x7f7f4c09d8d0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f7f58005fd0 tx=0x7f7f58005dc0 comp rx=0 tx=0).stop 2023-03-27T10:21:56.945 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.946+0000 7f7f69830700 1 -- 172.21.15.159:0/1885784286 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7f640ff8d0 msgr2=0x7f7f64196430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:56.946 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.946+0000 7f7f69830700 1 --2- 172.21.15.159:0/1885784286 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7f640ff8d0 0x7f7f64196430 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f7f5400c9e0 tx=0x7f7f5400cda0 comp rx=0 tx=0).stop 2023-03-27T10:21:56.946 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.946+0000 7f7f69830700 1 -- 172.21.15.159:0/1885784286 shutdown_connections 2023-03-27T10:21:56.946 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.946+0000 7f7f69830700 1 --2- 172.21.15.159:0/1885784286 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7f4c09b420 0x7f7f4c09d8d0 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:56.946 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.946+0000 7f7f69830700 1 --2- 172.21.15.159:0/1885784286 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7f640fef30 0x7f7f64195ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:56.946 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.946+0000 7f7f69830700 1 --2- 172.21.15.159:0/1885784286 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7f64103fd0 0x7f7f6419a8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:56.946 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.946+0000 7f7f69830700 1 --2- 172.21.15.159:0/1885784286 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7f640ff8d0 0x7f7f64196430 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:56.946 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.946+0000 7f7f69830700 1 -- 172.21.15.159:0/1885784286 >> 172.21.15.159:0/1885784286 conn(0x7f7f640fa7f0 msgr2=0x7f7f64068c10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:21:56.947 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.946+0000 7f7f69830700 1 -- 172.21.15.159:0/1885784286 shutdown_connections 2023-03-27T10:21:56.947 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:21:56.946+0000 7f7f69830700 1 -- 172.21.15.159:0/1885784286 wait complete. 2023-03-27T10:21:57.367 INFO:tasks.cephadm:Adding mds.a on smithi159 2023-03-27T10:21:57.367 INFO:tasks.cephadm:Adding mds.d on smithi159 2023-03-27T10:21:57.368 INFO:tasks.cephadm:Adding mds.g on smithi159 2023-03-27T10:21:57.368 INFO:tasks.cephadm:Adding mds.j on smithi159 2023-03-27T10:21:57.368 INFO:tasks.cephadm:Adding mds.b on smithi161 2023-03-27T10:21:57.368 INFO:tasks.cephadm:Adding mds.e on smithi161 2023-03-27T10:21:57.368 INFO:tasks.cephadm:Adding mds.h on smithi161 2023-03-27T10:21:57.369 INFO:tasks.cephadm:Adding mds.k on smithi161 2023-03-27T10:21:57.369 INFO:tasks.cephadm:Adding mds.c on smithi171 2023-03-27T10:21:57.369 INFO:tasks.cephadm:Adding mds.f on smithi171 2023-03-27T10:21:57.369 INFO:tasks.cephadm:Adding mds.i on smithi171 2023-03-27T10:21:57.369 INFO:tasks.cephadm:Adding mds.l on smithi171 2023-03-27T10:21:57.370 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph orch apply mds all '12;smithi159=a;smithi159=d;smithi159=g;smithi159=j;smithi161=b;smithi161=e;smithi161=h;smithi161=k;smithi171=c;smithi171=f;smithi171=i;smithi171=l' 2023-03-27T10:21:57.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:57 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/1885784286' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-03-27T10:21:57.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:57 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/1885784286' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-03-27T10:21:57.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:57 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/1885784286' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-03-27T10:21:58.025 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.c/config 2023-03-27T10:21:58.717 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:21:58 smithi171 ceph-mon[94808]: pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:21:58.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:21:58 smithi161 ceph-mon[94333]: pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:21:58.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:21:58 smithi159 ceph-mon[93039]: pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:21:59.413 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.414+0000 7fecaf4f0700 1 -- 172.21.15.171:0/1152576182 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7feca8100830 msgr2=0x7feca81070f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:59.414 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.414+0000 7fecaf4f0700 1 --2- 172.21.15.171:0/1152576182 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7feca8100830 0x7feca81070f0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7feca000b0a0 tx=0x7feca000b3b0 comp rx=0 tx=0).stop 2023-03-27T10:21:59.414 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.414+0000 7fecaf4f0700 1 -- 172.21.15.171:0/1152576182 shutdown_connections 2023-03-27T10:21:59.414 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.414+0000 7fecaf4f0700 1 --2- 172.21.15.171:0/1152576182 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7feca8100830 0x7feca81070f0 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:59.414 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.414+0000 7fecaf4f0700 1 --2- 172.21.15.171:0/1152576182 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7feca80feca0 0x7feca80ff0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:59.414 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.414+0000 7fecaf4f0700 1 --2- 172.21.15.171:0/1152576182 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feca80ffea0 0x7feca81002f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:59.414 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.414+0000 7fecaf4f0700 1 -- 172.21.15.171:0/1152576182 >> 172.21.15.171:0/1152576182 conn(0x7feca80fa2b0 msgr2=0x7feca80fc720 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:21:59.415 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.414+0000 7fecaf4f0700 1 -- 172.21.15.171:0/1152576182 shutdown_connections 2023-03-27T10:21:59.415 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.414+0000 7fecaf4f0700 1 -- 172.21.15.171:0/1152576182 wait complete. 2023-03-27T10:21:59.416 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.414+0000 7fecaf4f0700 1 Processor -- start 2023-03-27T10:21:59.416 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecaf4f0700 1 -- start start 2023-03-27T10:21:59.417 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecaf4f0700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feca80feca0 0x7feca8068c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:59.417 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecaf4f0700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7feca80ffea0 0x7feca819a900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:59.417 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecaf4f0700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7feca8100830 0x7feca819cf50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:59.417 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecaf4f0700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feca819d6f0 con 0x7feca80feca0 2023-03-27T10:21:59.417 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecaf4f0700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feca819d860 con 0x7feca8100830 2023-03-27T10:21:59.417 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecaf4f0700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feca819da20 con 0x7feca80ffea0 2023-03-27T10:21:59.417 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecaca8b700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7feca80ffea0 0x7feca819a900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:59.418 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecaca8b700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7feca80ffea0 0x7feca819a900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.171:3300/0 says I am v2:172.21.15.171:55590/0 (socket says 172.21.15.171:55590) 2023-03-27T10:21:59.418 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecaca8b700 1 -- 172.21.15.171:0/1139811210 learned_addr learned my addr 172.21.15.171:0/1139811210 (peer_addr_for_me v2:172.21.15.171:0/0) 2023-03-27T10:21:59.418 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecada8d700 1 --2- 172.21.15.171:0/1139811210 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7feca8100830 0x7feca819cf50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:59.418 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecad28c700 1 --2- 172.21.15.171:0/1139811210 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feca80feca0 0x7feca8068c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:59.418 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecaca8b700 1 -- 172.21.15.171:0/1139811210 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7feca8100830 msgr2=0x7feca819cf50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:59.418 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecaca8b700 1 --2- 172.21.15.171:0/1139811210 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7feca8100830 0x7feca819cf50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:59.418 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecaca8b700 1 -- 172.21.15.171:0/1139811210 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feca80feca0 msgr2=0x7feca8068c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:59.418 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecaca8b700 1 --2- 172.21.15.171:0/1139811210 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feca80feca0 0x7feca8068c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:59.419 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecaca8b700 1 -- 172.21.15.171:0/1139811210 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feca0009d00 con 0x7feca80ffea0 2023-03-27T10:21:59.419 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fecaca8b700 1 --2- 172.21.15.171:0/1139811210 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7feca80ffea0 0x7feca819a900 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fec9800e9a0 tx=0x7fec9800ed60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:21:59.419 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.415+0000 7fec9e7fc700 1 -- 172.21.15.171:0/1139811210 <== mon.1 v2:172.21.15.171:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec98009790 con 0x7feca80ffea0 2023-03-27T10:21:59.419 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.416+0000 7fec9e7fc700 1 -- 172.21.15.171:0/1139811210 <== mon.1 v2:172.21.15.171:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fec9800cbd0 con 0x7feca80ffea0 2023-03-27T10:21:59.419 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.416+0000 7fecaf4f0700 1 -- 172.21.15.171:0/1139811210 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feca810a600 con 0x7feca80ffea0 2023-03-27T10:21:59.419 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.416+0000 7fec9e7fc700 1 -- 172.21.15.171:0/1139811210 <== mon.1 v2:172.21.15.171:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec98018680 con 0x7feca80ffea0 2023-03-27T10:21:59.419 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.416+0000 7fecaf4f0700 1 -- 172.21.15.171:0/1139811210 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feca810a470 con 0x7feca80ffea0 2023-03-27T10:21:59.419 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.417+0000 7fecaf4f0700 1 -- 172.21.15.171:0/1139811210 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7feca819e070 con 0x7feca80ffea0 2023-03-27T10:21:59.420 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.418+0000 7fec9e7fc700 1 -- 172.21.15.171:0/1139811210 <== mon.1 v2:172.21.15.171:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7fec9800c040 con 0x7feca80ffea0 2023-03-27T10:21:59.420 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.418+0000 7fec9e7fc700 1 --2- 172.21.15.171:0/1139811210 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fec9409b3a0 0x7fec9409d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:21:59.421 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.418+0000 7fec9e7fc700 1 -- 172.21.15.171:0/1139811210 <== mon.1 v2:172.21.15.171:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7fec98014070 con 0x7feca80ffea0 2023-03-27T10:21:59.421 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.418+0000 7fecad28c700 1 --2- 172.21.15.171:0/1139811210 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fec9409b3a0 0x7fec9409d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:21:59.421 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.419+0000 7fecad28c700 1 --2- 172.21.15.171:0/1139811210 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fec9409b3a0 0x7fec9409d850 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7feca80ffd00 tx=0x7feca4009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:21:59.421 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.422+0000 7fec9e7fc700 1 -- 172.21.15.171:0/1139811210 <== mon.1 v2:172.21.15.171:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fec98090ff0 con 0x7feca80ffea0 2023-03-27T10:21:59.612 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.612+0000 7fecaf4f0700 1 -- 172.21.15.171:0/1139811210 --> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] -- mgr_command(tid 0: {"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi159=a;smithi159=d;smithi159=g;smithi159=j;smithi161=b;smithi161=e;smithi161=h;smithi161=k;smithi171=c;smithi171=f;smithi171=i;smithi171=l", "target": ["mon-mgr", ""]}) v1 -- 0x7feca8060c10 con 0x7fec9409b3a0 2023-03-27T10:21:59.621 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.622+0000 7fec9e7fc700 1 -- 172.21.15.171:0/1139811210 <== mgr.14152 v2:172.21.15.159:6800/3290710483 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+28 (secure 0 0 0) 0x7feca8060c10 con 0x7fec9409b3a0 2023-03-27T10:21:59.622 INFO:teuthology.orchestra.run.smithi171.stdout:Scheduled mds.all update... 2023-03-27T10:21:59.623 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.624+0000 7fecaf4f0700 1 -- 172.21.15.171:0/1139811210 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fec9409b3a0 msgr2=0x7fec9409d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:59.624 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.624+0000 7fecaf4f0700 1 --2- 172.21.15.171:0/1139811210 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fec9409b3a0 0x7fec9409d850 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7feca80ffd00 tx=0x7feca4009260 comp rx=0 tx=0).stop 2023-03-27T10:21:59.624 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.624+0000 7fecaf4f0700 1 -- 172.21.15.171:0/1139811210 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7feca80ffea0 msgr2=0x7feca819a900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:21:59.624 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.624+0000 7fecaf4f0700 1 --2- 172.21.15.171:0/1139811210 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7feca80ffea0 0x7feca819a900 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fec9800e9a0 tx=0x7fec9800ed60 comp rx=0 tx=0).stop 2023-03-27T10:21:59.624 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.624+0000 7fecaf4f0700 1 -- 172.21.15.171:0/1139811210 shutdown_connections 2023-03-27T10:21:59.624 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.624+0000 7fecaf4f0700 1 --2- 172.21.15.171:0/1139811210 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fec9409b3a0 0x7fec9409d850 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:59.624 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.624+0000 7fecaf4f0700 1 --2- 172.21.15.171:0/1139811210 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7feca8100830 0x7feca819cf50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:59.624 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.624+0000 7fecaf4f0700 1 --2- 172.21.15.171:0/1139811210 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7feca80feca0 0x7feca8068c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:59.625 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.624+0000 7fecaf4f0700 1 --2- 172.21.15.171:0/1139811210 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7feca80ffea0 0x7feca819a900 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:21:59.625 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.624+0000 7fecaf4f0700 1 -- 172.21.15.171:0/1139811210 >> 172.21.15.171:0/1139811210 conn(0x7feca80fa2b0 msgr2=0x7feca80fc500 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:21:59.625 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.624+0000 7fecaf4f0700 1 -- 172.21.15.171:0/1139811210 shutdown_connections 2023-03-27T10:21:59.625 INFO:teuthology.orchestra.run.smithi171.stderr:2023-03-27T10:21:59.624+0000 7fecaf4f0700 1 -- 172.21.15.171:0/1139811210 wait complete. 2023-03-27T10:22:00.082 DEBUG:teuthology.orchestra.run.smithi159:mds.a> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.a.service 2023-03-27T10:22:00.086 DEBUG:teuthology.orchestra.run.smithi159:mds.d> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.d.service 2023-03-27T10:22:00.123 INFO:journalctl@ceph.mds.a.smithi159.stdout:-- Logs begin at Mon 2023-03-27 10:03:12 UTC. -- 2023-03-27T10:22:00.124 DEBUG:teuthology.orchestra.run.smithi159:mds.g> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.g.service 2023-03-27T10:22:00.153 INFO:journalctl@ceph.mds.d.smithi159.stdout:-- Logs begin at Mon 2023-03-27 10:03:12 UTC. -- 2023-03-27T10:22:00.154 DEBUG:teuthology.orchestra.run.smithi159:mds.j> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.j.service 2023-03-27T10:22:00.181 INFO:journalctl@ceph.mds.g.smithi159.stdout:-- Logs begin at Mon 2023-03-27 10:03:12 UTC. -- 2023-03-27T10:22:00.182 DEBUG:teuthology.orchestra.run.smithi161:mds.b> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.b.service 2023-03-27T10:22:00.185 DEBUG:teuthology.orchestra.run.smithi161:mds.e> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.e.service 2023-03-27T10:22:00.209 INFO:journalctl@ceph.mds.j.smithi159.stdout:-- Logs begin at Mon 2023-03-27 10:03:12 UTC. -- 2023-03-27T10:22:00.222 INFO:journalctl@ceph.mds.b.smithi161.stdout:-- Logs begin at Mon 2023-03-27 10:02:59 UTC. -- 2023-03-27T10:22:00.223 DEBUG:teuthology.orchestra.run.smithi161:mds.h> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.h.service 2023-03-27T10:22:00.252 INFO:journalctl@ceph.mds.e.smithi161.stdout:-- Logs begin at Mon 2023-03-27 10:02:59 UTC. -- 2023-03-27T10:22:00.253 DEBUG:teuthology.orchestra.run.smithi161:mds.k> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.k.service 2023-03-27T10:22:00.281 INFO:journalctl@ceph.mds.h.smithi161.stdout:-- Logs begin at Mon 2023-03-27 10:02:59 UTC. -- 2023-03-27T10:22:00.282 DEBUG:teuthology.orchestra.run.smithi171:mds.c> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.c.service 2023-03-27T10:22:00.284 DEBUG:teuthology.orchestra.run.smithi171:mds.f> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.f.service 2023-03-27T10:22:00.308 INFO:journalctl@ceph.mds.k.smithi161.stdout:-- Logs begin at Mon 2023-03-27 10:02:59 UTC. -- 2023-03-27T10:22:00.313 INFO:journalctl@ceph.mds.c.smithi171.stdout:-- Logs begin at Mon 2023-03-27 10:03:15 UTC. -- 2023-03-27T10:22:00.314 DEBUG:teuthology.orchestra.run.smithi171:mds.i> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.i.service 2023-03-27T10:22:00.341 INFO:journalctl@ceph.mds.f.smithi171.stdout:-- Logs begin at Mon 2023-03-27 10:03:15 UTC. -- 2023-03-27T10:22:00.342 DEBUG:teuthology.orchestra.run.smithi171:mds.l> sudo journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.l.service 2023-03-27T10:22:00.367 INFO:journalctl@ceph.mds.i.smithi171.stdout:-- Logs begin at Mon 2023-03-27 10:03:15 UTC. -- 2023-03-27T10:22:00.368 INFO:tasks.cephadm:Setting up CephFS filesystem(s)... 2023-03-27T10:22:00.368 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-03-27T10:22:00.396 INFO:journalctl@ceph.mds.l.smithi171.stdout:-- Logs begin at Mon 2023-03-27 10:03:15 UTC. -- 2023-03-27T10:22:00.508 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.508+0000 7fccd7e1b700 1 -- 172.21.15.159:0/3909562033 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fccd00febd0 msgr2=0x7fccd00fefe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:00.508 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.508+0000 7fccd7e1b700 1 --2- 172.21.15.159:0/3909562033 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fccd00febd0 0x7fccd00fefe0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7fccc0009990 tx=0x7fccc0009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.508 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.508+0000 7fccd7e1b700 1 -- 172.21.15.159:0/3909562033 shutdown_connections 2023-03-27T10:22:00.508 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.508+0000 7fccd7e1b700 1 --2- 172.21.15.159:0/3909562033 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fccd0100810 0x7fccd01070b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.509 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.508+0000 7fccd7e1b700 1 --2- 172.21.15.159:0/3909562033 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fccd00febd0 0x7fccd00fefe0 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.509 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.508+0000 7fccd7e1b700 1 --2- 172.21.15.159:0/3909562033 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fccd00ffe60 0x7fccd01002d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.509 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.508+0000 7fccd7e1b700 1 -- 172.21.15.159:0/3909562033 >> 172.21.15.159:0/3909562033 conn(0x7fccd00fa210 msgr2=0x7fccd00fc620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:00.509 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.508+0000 7fccd7e1b700 1 -- 172.21.15.159:0/3909562033 shutdown_connections 2023-03-27T10:22:00.509 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.508+0000 7fccd7e1b700 1 -- 172.21.15.159:0/3909562033 wait complete. 2023-03-27T10:22:00.511 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.509+0000 7fccd7e1b700 1 Processor -- start 2023-03-27T10:22:00.511 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.509+0000 7fccd7e1b700 1 -- start start 2023-03-27T10:22:00.511 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.509+0000 7fccd7e1b700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fccd00febd0 0x7fccd01949b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:00.512 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.509+0000 7fccd7e1b700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fccd00ffe60 0x7fccd0194ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:00.512 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.509+0000 7fccd7e1b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fccd0100810 0x7fccd019bf60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:00.512 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.509+0000 7fccd7e1b700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fccd019c530 con 0x7fccd0100810 2023-03-27T10:22:00.512 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.509+0000 7fccd7e1b700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fccd019c670 con 0x7fccd00febd0 2023-03-27T10:22:00.512 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.509+0000 7fccd7e1b700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fccd019c800 con 0x7fccd00ffe60 2023-03-27T10:22:00.512 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.509+0000 7fccd63b8700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fccd0100810 0x7fccd019bf60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:00.512 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.509+0000 7fccd63b8700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fccd0100810 0x7fccd019bf60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45314/0 (socket says 172.21.15.159:45314) 2023-03-27T10:22:00.513 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.509+0000 7fccd63b8700 1 -- 172.21.15.159:0/3961411006 learned_addr learned my addr 172.21.15.159:0/3961411006 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:00.513 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.510+0000 7fccd53b6700 1 --2- 172.21.15.159:0/3961411006 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fccd00ffe60 0x7fccd0194ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:00.513 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.510+0000 7fccd5bb7700 1 --2- 172.21.15.159:0/3961411006 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fccd00febd0 0x7fccd01949b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:00.513 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.510+0000 7fccd63b8700 1 -- 172.21.15.159:0/3961411006 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fccd00ffe60 msgr2=0x7fccd0194ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:00.513 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.510+0000 7fccd63b8700 1 --2- 172.21.15.159:0/3961411006 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fccd00ffe60 0x7fccd0194ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.513 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.510+0000 7fccd63b8700 1 -- 172.21.15.159:0/3961411006 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fccd00febd0 msgr2=0x7fccd01949b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:00.513 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.510+0000 7fccd63b8700 1 --2- 172.21.15.159:0/3961411006 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fccd00febd0 0x7fccd01949b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.514 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.510+0000 7fccd63b8700 1 -- 172.21.15.159:0/3961411006 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fccc00095f0 con 0x7fccd0100810 2023-03-27T10:22:00.514 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.510+0000 7fccd53b6700 1 --2- 172.21.15.159:0/3961411006 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fccd00ffe60 0x7fccd0194ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:22:00.514 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.510+0000 7fccd5bb7700 1 --2- 172.21.15.159:0/3961411006 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fccd00febd0 0x7fccd01949b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:22:00.514 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.510+0000 7fccd63b8700 1 --2- 172.21.15.159:0/3961411006 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fccd0100810 0x7fccd019bf60 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7fccc8007ee0 tx=0x7fccc800d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:00.514 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.510+0000 7fccc6ffd700 1 -- 172.21.15.159:0/3961411006 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fccc8017070 con 0x7fccd0100810 2023-03-27T10:22:00.514 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.510+0000 7fccc6ffd700 1 -- 172.21.15.159:0/3961411006 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fccc800f040 con 0x7fccd0100810 2023-03-27T10:22:00.514 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.510+0000 7fccd7e1b700 1 -- 172.21.15.159:0/3961411006 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fccd010a580 con 0x7fccd0100810 2023-03-27T10:22:00.514 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.510+0000 7fccc6ffd700 1 -- 172.21.15.159:0/3961411006 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fccc8005740 con 0x7fccd0100810 2023-03-27T10:22:00.515 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.510+0000 7fccd7e1b700 1 -- 172.21.15.159:0/3961411006 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fccd010a3f0 con 0x7fccd0100810 2023-03-27T10:22:00.515 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.511+0000 7fccd7e1b700 1 -- 172.21.15.159:0/3961411006 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fccd0105bf0 con 0x7fccd0100810 2023-03-27T10:22:00.515 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.512+0000 7fccc6ffd700 1 -- 172.21.15.159:0/3961411006 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7fccc80058a0 con 0x7fccd0100810 2023-03-27T10:22:00.515 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.513+0000 7fccc6ffd700 1 --2- 172.21.15.159:0/3961411006 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fccbc09b3a0 0x7fccbc09d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:00.516 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.513+0000 7fccd5bb7700 1 --2- 172.21.15.159:0/3961411006 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fccbc09b3a0 0x7fccbc09d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:00.516 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.513+0000 7fccc6ffd700 1 -- 172.21.15.159:0/3961411006 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7fccc80c44b0 con 0x7fccd0100810 2023-03-27T10:22:00.516 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.513+0000 7fccd5bb7700 1 --2- 172.21.15.159:0/3961411006 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fccbc09b3a0 0x7fccbc09d850 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7fccc000b5c0 tx=0x7fccc0005fb0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:00.516 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.517+0000 7fccc6ffd700 1 -- 172.21.15.159:0/3961411006 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fccc80902c0 con 0x7fccd0100810 2023-03-27T10:22:00.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:00 smithi161 ceph-mon[94333]: pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:00.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:00.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:22:00.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:00.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:22:00.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:00.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:00.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:00.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:00.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:00 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:00.658 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:00 smithi171 ceph-mon[94808]: pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:00.659 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:00.659 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:22:00.659 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:00.659 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:22:00.659 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:00.659 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:00.659 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:00.660 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:00.660 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:00 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:00.701 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.701+0000 7fccd7e1b700 1 -- 172.21.15.159:0/3961411006 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fccd002cdb0 con 0x7fccd0100810 2023-03-27T10:22:00.701 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.702+0000 7fccc6ffd700 1 -- 172.21.15.159:0/3961411006 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v81) v1 ==== 74+0+18455 (secure 0 0 0) 0x7fccc80938e0 con 0x7fccd0100810 2023-03-27T10:22:00.702 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:00.702 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":81,"fsid":"2c3cd140-cc88-11ed-9afe-001a4aab830c","created":"2023-03-27T10:15:32.617230+0000","modified":"2023-03-27T10:21:50.939009+0000","last_up_change":"2023-03-27T10:21:49.442480+0000","last_in_change":"2023-03-27T10:21:32.265278+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-03-27T10:18:35.277335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f47f99b9-47f9-4d94-83d9-33b1278be17e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6803","nonce":87273908}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6805","nonce":87273908}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6809","nonce":87273908}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6807","nonce":87273908}]},"public_addr":"172.21.15.159:6803/87273908","cluster_addr":"172.21.15.159:6805/87273908","heartbeat_back_addr":"172.21.15.159:6809/87273908","heartbeat_front_addr":"172.21.15.159:6807/87273908","state":["exists","up"]},{"osd":1,"uuid":"97f66da9-22b4-4eb5-9c77-9d37cf0fca82","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6800","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6801","nonce":2094166586}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6802","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6803","nonce":2094166586}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6806","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6807","nonce":2094166586}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6804","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6805","nonce":2094166586}]},"public_addr":"172.21.15.161:6801/2094166586","cluster_addr":"172.21.15.161:6803/2094166586","heartbeat_back_addr":"172.21.15.161:6807/2094166586","heartbeat_front_addr":"172.21.15.161:6805/2094166586","state":["exists","up"]},{"osd":2,"uuid":"ba9f5f82-3aef-42f7-a9e6-0e07aa7d8df1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6801","nonce":2724827688}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6803","nonce":2724827688}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6807","nonce":2724827688}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6805","nonce":2724827688}]},"public_addr":"172.21.15.171:6801/2724827688","cluster_addr":"172.21.15.171:6803/2724827688","heartbeat_back_addr":"172.21.15.171:6807/2724827688","heartbeat_front_addr":"172.21.15.171:6805/2724827688","state":["exists","up"]},{"osd":3,"uuid":"4ef4b519-fdef-4d38-b387-bc14ea381bd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6811","nonce":1992276871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6813","nonce":1992276871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6817","nonce":1992276871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6815","nonce":1992276871}]},"public_addr":"172.21.15.159:6811/1992276871","cluster_addr":"172.21.15.159:6813/1992276871","heartbeat_back_addr":"172.21.15.159:6817/1992276871","heartbeat_front_addr":"172.21.15.159:6815/1992276871","state":["exists","up"]},{"osd":4,"uuid":"982675c1-dd87-46b4-97ef-a5dde1b1d777","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6808","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6809","nonce":3978602555}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6810","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6811","nonce":3978602555}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6814","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6815","nonce":3978602555}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6812","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6813","nonce":3978602555}]},"public_addr":"172.21.15.161:6809/3978602555","cluster_addr":"172.21.15.161:6811/3978602555","heartbeat_back_addr":"172.21.15.161:6815/3978602555","heartbeat_front_addr":"172.21.15.161:6813/3978602555","state":["exists","up"]},{"osd":5,"uuid":"b4de4a85-4f91-4194-a1ce-db12f1898c4f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6809","nonce":569611891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6811","nonce":569611891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6815","nonce":569611891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6813","nonce":569611891}]},"public_addr":"172.21.15.171:6809/569611891","cluster_addr":"172.21.15.171:6811/569611891","heartbeat_back_addr":"172.21.15.171:6815/569611891","heartbeat_front_addr":"172.21.15.171:6813/569611891","state":["exists","up"]},{"osd":6,"uuid":"dff7d5f4-655f-4baa-9084-03591a4b3bb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6819","nonce":1176978350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6821","nonce":1176978350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6825","nonce":1176978350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6823","nonce":1176978350}]},"public_addr":"172.21.15.159:6819/1176978350","cluster_addr":"172.21.15.159:6821/1176978350","heartbeat_back_addr":"172.21.15.159:6825/1176978350","heartbeat_front_addr":"172.21.15.159:6823/1176978350","state":["exists","up"]},{"osd":7,"uuid":"30f026d9-6963-4468-95e8-78c7ed2e89f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6816","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6817","nonce":2098174471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6818","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6819","nonce":2098174471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6822","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6823","nonce":2098174471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6820","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6821","nonce":2098174471}]},"public_addr":"172.21.15.161:6817/2098174471","cluster_addr":"172.21.15.161:6819/2098174471","heartbeat_back_addr":"172.21.15.161:6823/2098174471","heartbeat_front_addr":"172.21.15.161:6821/2098174471","state":["exists","up"]},{"osd":8,"uuid":"9f69fbcc-32ff-4d4b-b562-ee0a12745cc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6817","nonce":1033586906}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6819","nonce":1033586906}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6823","nonce":1033586906}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6821","nonce":1033586906}]},"public_addr":"172.21.15.171:6817/1033586906","cluster_addr":"172.21.15.171:6819/1033586906","heartbeat_back_addr":"172.21.15.171:6823/1033586906","heartbeat_front_addr":"172.21.15.171:6821/1033586906","state":["exists","up"]},{"osd":9,"uuid":"d7fb9eed-901b-4943-9529-6edcba2385e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6827","nonce":3285179829}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6829","nonce":3285179829}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6832","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6833","nonce":3285179829}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6831","nonce":3285179829}]},"public_addr":"172.21.15.159:6827/3285179829","cluster_addr":"172.21.15.159:6829/3285179829","heartbeat_back_addr":"172.21.15.159:6833/3285179829","heartbeat_front_addr":"172.21.15.159:6831/3285179829","state":["exists","up"]},{"osd":10,"uuid":"4871ca03-30ae-4e87-942c-9441dd31fb40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6824","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6825","nonce":946677932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6826","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6827","nonce":946677932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6830","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6831","nonce":946677932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6828","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6829","nonce":946677932}]},"public_addr":"172.21.15.161:6825/946677932","cluster_addr":"172.21.15.161:6827/946677932","heartbeat_back_addr":"172.21.15.161:6831/946677932","heartbeat_front_addr":"172.21.15.161:6829/946677932","state":["exists","up"]},{"osd":11,"uuid":"1fb37d9a-ed52-4c0d-a082-65e17d242106","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6825","nonce":596725174}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6827","nonce":596725174}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6831","nonce":596725174}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6829","nonce":596725174}]},"public_addr":"172.21.15.171:6825/596725174","cluster_addr":"172.21.15.171:6827/596725174","heartbeat_back_addr":"172.21.15.171:6831/596725174","heartbeat_front_addr":"172.21.15.171:6829/596725174","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:17:49.128592+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:10.944593+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:32.989432+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:54.675294+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:16.543485+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:38.390326+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:59.566202+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:20:21.348420+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:20:43.383576+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:04.710992+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:26.286573+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:48.277317+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.159:0/653469093":"2023-03-28T10:16:26.900382+0000","172.21.15.159:6800/3561797228":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/3045586257":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/2859804517":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6801/3561797228":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/3487813906":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/1306320900":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6800/1729835725":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6801/1729835725":"2023-03-28T10:16:04.737100+0000","172.21.15.159:0/72206061":"2023-03-28T10:16:04.737100+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-03-27T10:22:00.704 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.704+0000 7fccd7e1b700 1 -- 172.21.15.159:0/3961411006 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fccbc09b3a0 msgr2=0x7fccbc09d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:00.704 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.704+0000 7fccd7e1b700 1 --2- 172.21.15.159:0/3961411006 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fccbc09b3a0 0x7fccbc09d850 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7fccc000b5c0 tx=0x7fccc0005fb0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.704 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.704+0000 7fccd7e1b700 1 -- 172.21.15.159:0/3961411006 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fccd0100810 msgr2=0x7fccd019bf60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:00.704 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.704+0000 7fccd7e1b700 1 --2- 172.21.15.159:0/3961411006 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fccd0100810 0x7fccd019bf60 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7fccc8007ee0 tx=0x7fccc800d3b0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.704 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.704+0000 7fccd7e1b700 1 -- 172.21.15.159:0/3961411006 shutdown_connections 2023-03-27T10:22:00.704 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.704+0000 7fccd7e1b700 1 --2- 172.21.15.159:0/3961411006 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fccbc09b3a0 0x7fccbc09d850 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.705 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.704+0000 7fccd7e1b700 1 --2- 172.21.15.159:0/3961411006 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fccd0100810 0x7fccd019bf60 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.705 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.704+0000 7fccd7e1b700 1 --2- 172.21.15.159:0/3961411006 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fccd00febd0 0x7fccd01949b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.705 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.704+0000 7fccd7e1b700 1 --2- 172.21.15.159:0/3961411006 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fccd00ffe60 0x7fccd0194ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.705 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.704+0000 7fccd7e1b700 1 -- 172.21.15.159:0/3961411006 >> 172.21.15.159:0/3961411006 conn(0x7fccd00fa210 msgr2=0x7fccd00fc4a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:00.705 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.705+0000 7fccd7e1b700 1 -- 172.21.15.159:0/3961411006 shutdown_connections 2023-03-27T10:22:00.705 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.705+0000 7fccd7e1b700 1 -- 172.21.15.159:0/3961411006 wait complete. 2023-03-27T10:22:00.715 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:00 smithi159 ceph-mon[93039]: pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:00.715 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:00.716 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:22:00.716 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:00.716 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:22:00.716 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:00.716 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:00.716 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:00.716 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:00.717 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:00 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:00.718 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-03-27T10:18:35.277335+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2023-03-27T10:22:00.718 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2023-03-27T10:22:00.842 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.843+0000 7f4afaab5700 1 -- 172.21.15.159:0/1019945108 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4af40ffe60 msgr2=0x7f4af41002d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:00.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.843+0000 7f4afaab5700 1 --2- 172.21.15.159:0/1019945108 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4af40ffe60 0x7f4af41002d0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f4ae0009990 tx=0x7f4ae0009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.843+0000 7f4afaab5700 1 -- 172.21.15.159:0/1019945108 shutdown_connections 2023-03-27T10:22:00.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.843+0000 7f4afaab5700 1 --2- 172.21.15.159:0/1019945108 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f4af4100810 0x7f4af41070b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.843+0000 7f4afaab5700 1 --2- 172.21.15.159:0/1019945108 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f4af40febd0 0x7f4af40fefe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.843+0000 7f4afaab5700 1 --2- 172.21.15.159:0/1019945108 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4af40ffe60 0x7f4af41002d0 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.843+0000 7f4afaab5700 1 -- 172.21.15.159:0/1019945108 >> 172.21.15.159:0/1019945108 conn(0x7f4af40fa210 msgr2=0x7f4af40fc620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:00.844 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.843+0000 7f4afaab5700 1 -- 172.21.15.159:0/1019945108 shutdown_connections 2023-03-27T10:22:00.844 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.843+0000 7f4afaab5700 1 -- 172.21.15.159:0/1019945108 wait complete. 2023-03-27T10:22:00.844 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.843+0000 7f4afaab5700 1 Processor -- start 2023-03-27T10:22:00.844 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.843+0000 7f4afaab5700 1 -- start start 2023-03-27T10:22:00.846 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4afaab5700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4af40febd0 0x7f4af4195a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:00.846 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4afaab5700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f4af40ffe60 0x7f4af4195f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:00.846 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4afaab5700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f4af4100810 0x7f4af419cfc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:00.846 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4afaab5700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4af419d590 con 0x7f4af40febd0 2023-03-27T10:22:00.846 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4afaab5700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4af419d6d0 con 0x7f4af40ffe60 2023-03-27T10:22:00.846 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4afaab5700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4af419d860 con 0x7f4af4100810 2023-03-27T10:22:00.846 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4af8851700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4af40febd0 0x7f4af4195a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:00.847 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4af8851700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4af40febd0 0x7f4af4195a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45324/0 (socket says 172.21.15.159:45324) 2023-03-27T10:22:00.847 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4af8851700 1 -- 172.21.15.159:0/2423905122 learned_addr learned my addr 172.21.15.159:0/2423905122 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:00.847 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4af3fff700 1 --2- 172.21.15.159:0/2423905122 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f4af40ffe60 0x7f4af4195f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:00.847 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4af9052700 1 --2- 172.21.15.159:0/2423905122 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f4af4100810 0x7f4af419cfc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:00.847 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4af8851700 1 -- 172.21.15.159:0/2423905122 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f4af4100810 msgr2=0x7f4af419cfc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:00.847 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4af8851700 1 --2- 172.21.15.159:0/2423905122 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f4af4100810 0x7f4af419cfc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.847 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4af8851700 1 -- 172.21.15.159:0/2423905122 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f4af40ffe60 msgr2=0x7f4af4195f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:00.848 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4af8851700 1 --2- 172.21.15.159:0/2423905122 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f4af40ffe60 0x7f4af4195f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:00.848 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4af8851700 1 -- 172.21.15.159:0/2423905122 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4ae00095f0 con 0x7f4af40febd0 2023-03-27T10:22:00.848 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4af8851700 1 --2- 172.21.15.159:0/2423905122 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4af40febd0 0x7f4af4195a10 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f4ae800d950 tx=0x7f4ae800dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:00.848 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4af1ffb700 1 -- 172.21.15.159:0/2423905122 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ae8009940 con 0x7f4af40febd0 2023-03-27T10:22:00.848 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4af1ffb700 1 -- 172.21.15.159:0/2423905122 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4ae8010460 con 0x7f4af40febd0 2023-03-27T10:22:00.848 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4af1ffb700 1 -- 172.21.15.159:0/2423905122 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ae800f620 con 0x7f4af40febd0 2023-03-27T10:22:00.848 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4afaab5700 1 -- 172.21.15.159:0/2423905122 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4af410a530 con 0x7f4af40febd0 2023-03-27T10:22:00.848 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.844+0000 7f4afaab5700 1 -- 172.21.15.159:0/2423905122 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4af410a3a0 con 0x7f4af40febd0 2023-03-27T10:22:00.849 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.846+0000 7f4afaab5700 1 -- 172.21.15.159:0/2423905122 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4af4106da0 con 0x7f4af40febd0 2023-03-27T10:22:00.849 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.846+0000 7f4af1ffb700 1 -- 172.21.15.159:0/2423905122 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f4ae8022070 con 0x7f4af40febd0 2023-03-27T10:22:00.849 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.847+0000 7f4af1ffb700 1 --2- 172.21.15.159:0/2423905122 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f4ae409b4a0 0x7f4ae409d950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:00.849 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.847+0000 7f4af3fff700 1 --2- 172.21.15.159:0/2423905122 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f4ae409b4a0 0x7f4ae409d950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:00.850 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.847+0000 7f4af1ffb700 1 -- 172.21.15.159:0/2423905122 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f4ae80c4840 con 0x7f4af40febd0 2023-03-27T10:22:00.850 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.847+0000 7f4af3fff700 1 --2- 172.21.15.159:0/2423905122 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f4ae409b4a0 0x7f4ae409d950 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f4ae000b5c0 tx=0x7f4ae0005fd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:00.850 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:00.851+0000 7f4af1ffb700 1 -- 172.21.15.159:0/2423905122 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f4ae8090560 con 0x7f4af40febd0 2023-03-27T10:22:01.033 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.034+0000 7f4afaab5700 1 -- 172.21.15.159:0/2423905122 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f4af4061690 con 0x7f4af40febd0 2023-03-27T10:22:01.034 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.034+0000 7f4af1ffb700 1 -- 172.21.15.159:0/2423905122 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v81) v1 ==== 93+0+10 (secure 0 0 0) 0x7f4ae80160a0 con 0x7f4af40febd0 2023-03-27T10:22:01.034 INFO:teuthology.orchestra.run.smithi159.stdout:pg_num: 1 2023-03-27T10:22:01.036 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.036+0000 7f4afaab5700 1 -- 172.21.15.159:0/2423905122 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f4ae409b4a0 msgr2=0x7f4ae409d950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:01.036 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.036+0000 7f4afaab5700 1 --2- 172.21.15.159:0/2423905122 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f4ae409b4a0 0x7f4ae409d950 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f4ae000b5c0 tx=0x7f4ae0005fd0 comp rx=0 tx=0).stop 2023-03-27T10:22:01.036 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.037+0000 7f4afaab5700 1 -- 172.21.15.159:0/2423905122 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4af40febd0 msgr2=0x7f4af4195a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:01.036 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.037+0000 7f4afaab5700 1 --2- 172.21.15.159:0/2423905122 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4af40febd0 0x7f4af4195a10 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f4ae800d950 tx=0x7f4ae800dc60 comp rx=0 tx=0).stop 2023-03-27T10:22:01.037 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.037+0000 7f4afaab5700 1 -- 172.21.15.159:0/2423905122 shutdown_connections 2023-03-27T10:22:01.037 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.037+0000 7f4afaab5700 1 --2- 172.21.15.159:0/2423905122 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f4ae409b4a0 0x7f4ae409d950 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:01.037 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.037+0000 7f4afaab5700 1 --2- 172.21.15.159:0/2423905122 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f4af4100810 0x7f4af419cfc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:01.037 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.037+0000 7f4afaab5700 1 --2- 172.21.15.159:0/2423905122 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4af40febd0 0x7f4af4195a10 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:01.037 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.037+0000 7f4afaab5700 1 --2- 172.21.15.159:0/2423905122 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f4af40ffe60 0x7f4af4195f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:01.037 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.037+0000 7f4afaab5700 1 -- 172.21.15.159:0/2423905122 >> 172.21.15.159:0/2423905122 conn(0x7f4af40fa210 msgr2=0x7f4af40fc410 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:01.037 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.037+0000 7f4afaab5700 1 -- 172.21.15.159:0/2423905122 shutdown_connections 2023-03-27T10:22:01.037 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.037+0000 7f4afaab5700 1 -- 172.21.15.159:0/2423905122 wait complete. 2023-03-27T10:22:01.048 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-03-27T10:22:01.170 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.170+0000 7facadb7f700 1 -- 172.21.15.159:0/897915108 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7faca8100790 msgr2=0x7faca8107050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:01.170 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.170+0000 7facadb7f700 1 --2- 172.21.15.159:0/897915108 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7faca8100790 0x7faca8107050 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7faca000b0a0 tx=0x7faca000b3b0 comp rx=0 tx=0).stop 2023-03-27T10:22:01.170 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.170+0000 7facadb7f700 1 -- 172.21.15.159:0/897915108 shutdown_connections 2023-03-27T10:22:01.170 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.170+0000 7facadb7f700 1 --2- 172.21.15.159:0/897915108 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7faca80fec10 0x7faca80ff020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:01.170 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.170+0000 7facadb7f700 1 --2- 172.21.15.159:0/897915108 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7faca8100790 0x7faca8107050 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:01.171 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.170+0000 7facadb7f700 1 --2- 172.21.15.159:0/897915108 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7faca80ffe00 0x7faca8100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:01.171 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.170+0000 7facadb7f700 1 -- 172.21.15.159:0/897915108 >> 172.21.15.159:0/897915108 conn(0x7faca80fa1f0 msgr2=0x7faca80fc660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:01.171 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.170+0000 7facadb7f700 1 -- 172.21.15.159:0/897915108 shutdown_connections 2023-03-27T10:22:01.171 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.170+0000 7facadb7f700 1 -- 172.21.15.159:0/897915108 wait complete. 2023-03-27T10:22:01.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.171+0000 7facadb7f700 1 Processor -- start 2023-03-27T10:22:01.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.171+0000 7facadb7f700 1 -- start start 2023-03-27T10:22:01.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.171+0000 7facadb7f700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7faca80fec10 0x7faca80696f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:01.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.171+0000 7facadb7f700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7faca80ffe00 0x7faca819cd50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:01.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.171+0000 7facadb7f700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7faca8100790 0x7faca819f2a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:01.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.171+0000 7facadb7f700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faca819f980 con 0x7faca80fec10 2023-03-27T10:22:01.174 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.171+0000 7facadb7f700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faca819faf0 con 0x7faca8100790 2023-03-27T10:22:01.174 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.171+0000 7facadb7f700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faca819fcb0 con 0x7faca80ffe00 2023-03-27T10:22:01.174 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7faca77fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7faca80fec10 0x7faca80696f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:01.174 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7faca77fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7faca80fec10 0x7faca80696f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45338/0 (socket says 172.21.15.159:45338) 2023-03-27T10:22:01.174 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7faca77fe700 1 -- 172.21.15.159:0/4236807884 learned_addr learned my addr 172.21.15.159:0/4236807884 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:01.174 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7faca6ffd700 1 --2- 172.21.15.159:0/4236807884 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7faca80ffe00 0x7faca819cd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:01.174 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7faca7fff700 1 --2- 172.21.15.159:0/4236807884 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7faca8100790 0x7faca819f2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:01.174 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7faca77fe700 1 -- 172.21.15.159:0/4236807884 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7faca80ffe00 msgr2=0x7faca819cd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:01.175 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7faca77fe700 1 --2- 172.21.15.159:0/4236807884 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7faca80ffe00 0x7faca819cd50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:01.175 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7faca77fe700 1 -- 172.21.15.159:0/4236807884 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7faca8100790 msgr2=0x7faca819f2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:01.175 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7faca77fe700 1 --2- 172.21.15.159:0/4236807884 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7faca8100790 0x7faca819f2a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:01.175 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7faca77fe700 1 -- 172.21.15.159:0/4236807884 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faca0009d00 con 0x7faca80fec10 2023-03-27T10:22:01.175 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7faca6ffd700 1 --2- 172.21.15.159:0/4236807884 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7faca80ffe00 0x7faca819cd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:22:01.175 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7faca77fe700 1 --2- 172.21.15.159:0/4236807884 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7faca80fec10 0x7faca80696f0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7fac9800e970 tx=0x7fac9800ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:01.175 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7faca4ff9700 1 -- 172.21.15.159:0/4236807884 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fac98009790 con 0x7faca80fec10 2023-03-27T10:22:01.175 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7faca4ff9700 1 -- 172.21.15.159:0/4236807884 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fac980044e0 con 0x7faca80fec10 2023-03-27T10:22:01.176 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7facadb7f700 1 -- 172.21.15.159:0/4236807884 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faca810a580 con 0x7faca80fec10 2023-03-27T10:22:01.176 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7faca4ff9700 1 -- 172.21.15.159:0/4236807884 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fac98003e30 con 0x7faca80fec10 2023-03-27T10:22:01.176 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.172+0000 7facadb7f700 1 -- 172.21.15.159:0/4236807884 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faca810a3f0 con 0x7faca80fec10 2023-03-27T10:22:01.176 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.173+0000 7facadb7f700 1 -- 172.21.15.159:0/4236807884 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7faca8106220 con 0x7faca80fec10 2023-03-27T10:22:01.177 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.174+0000 7faca4ff9700 1 -- 172.21.15.159:0/4236807884 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7fac9801e020 con 0x7faca80fec10 2023-03-27T10:22:01.177 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.175+0000 7faca4ff9700 1 --2- 172.21.15.159:0/4236807884 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fac9009b3a0 0x7fac9009d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:01.177 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.175+0000 7faca6ffd700 1 --2- 172.21.15.159:0/4236807884 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fac9009b3a0 0x7fac9009d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:01.177 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.175+0000 7faca4ff9700 1 -- 172.21.15.159:0/4236807884 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7fac98014070 con 0x7faca80fec10 2023-03-27T10:22:01.177 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.175+0000 7faca6ffd700 1 --2- 172.21.15.159:0/4236807884 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fac9009b3a0 0x7fac9009d850 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7fac9c009c90 tx=0x7fac9c009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:01.178 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.178+0000 7faca4ff9700 1 -- 172.21.15.159:0/4236807884 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fac98090cb0 con 0x7faca80fec10 2023-03-27T10:22:01.226 INFO:journalctl@ceph.mds.l.smithi171.stdout:Mar 27 10:22:01 smithi171 systemd[1]: Starting Ceph mds.l for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:01.404 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.405+0000 7facadb7f700 1 -- 172.21.15.159:0/4236807884 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7faca80613e0 con 0x7faca80fec10 2023-03-27T10:22:01.405 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.405+0000 7faca4ff9700 1 -- 172.21.15.159:0/4236807884 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 1 v1) v1 ==== 93+0+501 (secure 0 0 0) 0x7faca80613e0 con 0x7faca80fec10 2023-03-27T10:22:01.405 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:01.405 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":1,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[]} 2023-03-27T10:22:01.407 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.407+0000 7facadb7f700 1 -- 172.21.15.159:0/4236807884 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fac9009b3a0 msgr2=0x7fac9009d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:01.407 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.407+0000 7facadb7f700 1 --2- 172.21.15.159:0/4236807884 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fac9009b3a0 0x7fac9009d850 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7fac9c009c90 tx=0x7fac9c009260 comp rx=0 tx=0).stop 2023-03-27T10:22:01.407 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.407+0000 7facadb7f700 1 -- 172.21.15.159:0/4236807884 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7faca80fec10 msgr2=0x7faca80696f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:01.407 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.407+0000 7facadb7f700 1 --2- 172.21.15.159:0/4236807884 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7faca80fec10 0x7faca80696f0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7fac9800e970 tx=0x7fac9800ed30 comp rx=0 tx=0).stop 2023-03-27T10:22:01.407 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.407+0000 7facadb7f700 1 -- 172.21.15.159:0/4236807884 shutdown_connections 2023-03-27T10:22:01.407 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.407+0000 7facadb7f700 1 --2- 172.21.15.159:0/4236807884 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fac9009b3a0 0x7fac9009d850 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:01.407 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.407+0000 7facadb7f700 1 --2- 172.21.15.159:0/4236807884 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7faca8100790 0x7faca819f2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:01.408 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.407+0000 7facadb7f700 1 --2- 172.21.15.159:0/4236807884 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7faca80fec10 0x7faca80696f0 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:01.408 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.407+0000 7facadb7f700 1 --2- 172.21.15.159:0/4236807884 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7faca80ffe00 0x7faca819cd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:01.408 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.407+0000 7facadb7f700 1 -- 172.21.15.159:0/4236807884 >> 172.21.15.159:0/4236807884 conn(0x7faca80fa1f0 msgr2=0x7faca8103360 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:01.408 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.407+0000 7facadb7f700 1 -- 172.21.15.159:0/4236807884 shutdown_connections 2023-03-27T10:22:01.408 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:01.407+0000 7facadb7f700 1 -- 172.21.15.159:0/4236807884 wait complete. 2023-03-27T10:22:01.408 INFO:teuthology.orchestra.run.smithi159.stderr:dumped fsmap epoch 1 2023-03-27T10:22:01.703 INFO:journalctl@ceph.mds.l.smithi171.stdout:Mar 27 10:22:01 smithi171 podman[119874]: 2023-03-27T10:22:01.703 INFO:journalctl@ceph.mds.l.smithi171.stdout:Mar 27 10:22:01 smithi171 ceph-mds[119914]: -- 172.21.15.171:0/1206591616 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f695a7cb60 con 0x55f695a52800 2023-03-27T10:22:01.703 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:01 smithi171 ceph-mon[94808]: from='client.24403 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi159=a;smithi159=d;smithi159=g;smithi159=j;smithi161=b;smithi161=e;smithi161=h;smithi161=k;smithi171=c;smithi171=f;smithi171=i;smithi171=l", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:22:01.704 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:01 smithi171 ceph-mon[94808]: Saving service mds.all spec with placement smithi159=a;smithi159=d;smithi159=g;smithi159=j;smithi161=b;smithi161=e;smithi161=h;smithi161=k;smithi171=c;smithi171=f;smithi171=i;smithi171=l;count:12 2023-03-27T10:22:01.704 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:01 smithi171 ceph-mon[94808]: Deploying daemon mds.l on smithi171 2023-03-27T10:22:01.704 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:01 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3961411006' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:22:01.704 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:01 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/2423905122' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-03-27T10:22:01.704 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:01 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/4236807884' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:01.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:01 smithi161 ceph-mon[94333]: from='client.24403 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi159=a;smithi159=d;smithi159=g;smithi159=j;smithi161=b;smithi161=e;smithi161=h;smithi161=k;smithi171=c;smithi171=f;smithi171=i;smithi171=l", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:22:01.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:01 smithi161 ceph-mon[94333]: Saving service mds.all spec with placement smithi159=a;smithi159=d;smithi159=g;smithi159=j;smithi161=b;smithi161=e;smithi161=h;smithi161=k;smithi171=c;smithi171=f;smithi171=i;smithi171=l;count:12 2023-03-27T10:22:01.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:01 smithi161 ceph-mon[94333]: Deploying daemon mds.l on smithi171 2023-03-27T10:22:01.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:01 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3961411006' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:22:01.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:01 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/2423905122' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-03-27T10:22:01.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:01 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/4236807884' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:01.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:01 smithi159 ceph-mon[93039]: from='client.24403 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi159=a;smithi159=d;smithi159=g;smithi159=j;smithi161=b;smithi161=e;smithi161=h;smithi161=k;smithi171=c;smithi171=f;smithi171=i;smithi171=l", "target": ["mon-mgr", ""]}]: dispatch 2023-03-27T10:22:01.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:01 smithi159 ceph-mon[93039]: Saving service mds.all spec with placement smithi159=a;smithi159=d;smithi159=g;smithi159=j;smithi161=b;smithi161=e;smithi161=h;smithi161=k;smithi171=c;smithi171=f;smithi171=i;smithi171=l;count:12 2023-03-27T10:22:01.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:01 smithi159 ceph-mon[93039]: Deploying daemon mds.l on smithi171 2023-03-27T10:22:01.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:01 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3961411006' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:22:01.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:01 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/2423905122' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-03-27T10:22:01.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:01 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/4236807884' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:01.976 INFO:journalctl@ceph.mds.l.smithi171.stdout:Mar 27 10:22:01 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-l[119890]: starting mds.l at 2023-03-27T10:22:01.976 INFO:journalctl@ceph.mds.l.smithi171.stdout:Mar 27 10:22:01 smithi171 bash[119874]: faee83e52d6787afc884fe064abe3e8670f4ba07832baf8b1934e3798992a19c 2023-03-27T10:22:01.976 INFO:journalctl@ceph.mds.l.smithi171.stdout:Mar 27 10:22:01 smithi171 systemd[1]: Started Ceph mds.l for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:22:02.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:02 smithi171 ceph-mon[94808]: pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:02.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:02 smithi171 ceph-mon[94808]: mds.? [v2:172.21.15.171:6832/1022778932,v1:172.21.15.171:6833/1022778932] up:boot 2023-03-27T10:22:02.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:02 smithi171 ceph-mon[94808]: fsmap 1 up:standby 2023-03-27T10:22:02.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:02 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2023-03-27T10:22:02.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:02 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:02.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:02 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:02.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:02 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:02.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:02 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:02.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:02 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:02.798 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:02 smithi161 ceph-mon[94333]: pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:02.799 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:02 smithi161 ceph-mon[94333]: mds.? [v2:172.21.15.171:6832/1022778932,v1:172.21.15.171:6833/1022778932] up:boot 2023-03-27T10:22:02.799 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:02 smithi161 ceph-mon[94333]: fsmap 1 up:standby 2023-03-27T10:22:02.799 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:02 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2023-03-27T10:22:02.799 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:02 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:02.799 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:02 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:02.800 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:02 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:02.800 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:02 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:02.800 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:02 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:02.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:02 smithi159 ceph-mon[93039]: pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:02.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:02 smithi159 ceph-mon[93039]: mds.? [v2:172.21.15.171:6832/1022778932,v1:172.21.15.171:6833/1022778932] up:boot 2023-03-27T10:22:02.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:02 smithi159 ceph-mon[93039]: fsmap 1 up:standby 2023-03-27T10:22:02.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:02 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2023-03-27T10:22:02.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:02 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:02.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:02 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:02.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:02 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:02.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:02 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:02.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:02 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:03.300 INFO:journalctl@ceph.mds.k.smithi161.stdout:Mar 27 10:22:03 smithi161 systemd[1]: Starting Ceph mds.k for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:03.420 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-03-27T10:22:03.555 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.556+0000 7fa730944700 1 -- 172.21.15.159:0/2810307223 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa72c0ffe60 msgr2=0x7fa72c1002d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:03.555 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.556+0000 7fa730944700 1 --2- 172.21.15.159:0/2810307223 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa72c0ffe60 0x7fa72c1002d0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7fa720009990 tx=0x7fa720009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:03.556 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.556+0000 7fa730944700 1 -- 172.21.15.159:0/2810307223 shutdown_connections 2023-03-27T10:22:03.556 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.556+0000 7fa730944700 1 --2- 172.21.15.159:0/2810307223 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa72c100810 0x7fa72c1070b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:03.556 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.556+0000 7fa730944700 1 --2- 172.21.15.159:0/2810307223 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa72c0febd0 0x7fa72c0fefe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:03.556 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.556+0000 7fa730944700 1 --2- 172.21.15.159:0/2810307223 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa72c0ffe60 0x7fa72c1002d0 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:03.556 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.556+0000 7fa730944700 1 -- 172.21.15.159:0/2810307223 >> 172.21.15.159:0/2810307223 conn(0x7fa72c0fa210 msgr2=0x7fa72c0fc620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:03.556 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.556+0000 7fa730944700 1 -- 172.21.15.159:0/2810307223 shutdown_connections 2023-03-27T10:22:03.556 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.556+0000 7fa730944700 1 -- 172.21.15.159:0/2810307223 wait complete. 2023-03-27T10:22:03.560 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.556+0000 7fa730944700 1 Processor -- start 2023-03-27T10:22:03.561 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.556+0000 7fa730944700 1 -- start start 2023-03-27T10:22:03.561 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa730944700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa72c0febd0 0x7fa72c068430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:03.561 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa730944700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa72c0ffe60 0x7fa72c19a790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:03.562 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa730944700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa72c100810 0x7fa72c19cdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:03.562 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa730944700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa72c19d570 con 0x7fa72c100810 2023-03-27T10:22:03.562 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa730944700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa72c19d6e0 con 0x7fa72c0ffe60 2023-03-27T10:22:03.563 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa730944700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa72c19d8a0 con 0x7fa72c0febd0 2023-03-27T10:22:03.563 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa72ad9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa72c100810 0x7fa72c19cdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:03.563 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa72ad9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa72c100810 0x7fa72c19cdd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45356/0 (socket says 172.21.15.159:45356) 2023-03-27T10:22:03.563 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa72ad9d700 1 -- 172.21.15.159:0/361447620 learned_addr learned my addr 172.21.15.159:0/361447620 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:03.564 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa72a59c700 1 --2- 172.21.15.159:0/361447620 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa72c0febd0 0x7fa72c068430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:03.564 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa729d9b700 1 --2- 172.21.15.159:0/361447620 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa72c0ffe60 0x7fa72c19a790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:03.564 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa72ad9d700 1 -- 172.21.15.159:0/361447620 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa72c0febd0 msgr2=0x7fa72c068430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:03.564 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa72ad9d700 1 --2- 172.21.15.159:0/361447620 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa72c0febd0 0x7fa72c068430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:03.565 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa72ad9d700 1 -- 172.21.15.159:0/361447620 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa72c0ffe60 msgr2=0x7fa72c19a790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:03.565 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa72ad9d700 1 --2- 172.21.15.159:0/361447620 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa72c0ffe60 0x7fa72c19a790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:03.565 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa72ad9d700 1 -- 172.21.15.159:0/361447620 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa7200095f0 con 0x7fa72c100810 2023-03-27T10:22:03.565 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa72a59c700 1 --2- 172.21.15.159:0/361447620 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa72c0febd0 0x7fa72c068430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:22:03.566 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa72ad9d700 1 --2- 172.21.15.159:0/361447620 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa72c100810 0x7fa72c19cdd0 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7fa72400ada0 tx=0x7fa72400c330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:03.566 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa71b7fe700 1 -- 172.21.15.159:0/361447620 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa72400f070 con 0x7fa72c100810 2023-03-27T10:22:03.566 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa71b7fe700 1 -- 172.21.15.159:0/361447620 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa724007b30 con 0x7fa72c100810 2023-03-27T10:22:03.566 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa730944700 1 -- 172.21.15.159:0/361447620 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa72c10a530 con 0x7fa72c100810 2023-03-27T10:22:03.567 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa71b7fe700 1 -- 172.21.15.159:0/361447620 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa724005440 con 0x7fa72c100810 2023-03-27T10:22:03.567 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.557+0000 7fa730944700 1 -- 172.21.15.159:0/361447620 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa72c10a3a0 con 0x7fa72c100810 2023-03-27T10:22:03.567 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.559+0000 7fa730944700 1 -- 172.21.15.159:0/361447620 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa72c105c20 con 0x7fa72c100810 2023-03-27T10:22:03.567 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.559+0000 7fa71b7fe700 1 -- 172.21.15.159:0/361447620 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7fa724019020 con 0x7fa72c100810 2023-03-27T10:22:03.567 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.560+0000 7fa71b7fe700 1 --2- 172.21.15.159:0/361447620 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa71409b3a0 0x7fa71409d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:03.567 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.560+0000 7fa72a59c700 1 --2- 172.21.15.159:0/361447620 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa71409b3a0 0x7fa71409d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:03.567 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.560+0000 7fa71b7fe700 1 -- 172.21.15.159:0/361447620 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7fa7240c3f90 con 0x7fa72c100810 2023-03-27T10:22:03.568 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.560+0000 7fa72a59c700 1 --2- 172.21.15.159:0/361447620 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa71409b3a0 0x7fa71409d850 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fa71c005fd0 tx=0x7fa71c005f60 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:03.568 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.564+0000 7fa71b7fe700 1 -- 172.21.15.159:0/361447620 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fa72408fe30 con 0x7fa72c100810 2023-03-27T10:22:03.721 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:03 smithi161 ceph-mon[94333]: Deploying daemon mds.k on smithi161 2023-03-27T10:22:03.721 INFO:journalctl@ceph.mds.k.smithi161.stdout:Mar 27 10:22:03 smithi161 podman[118384]: 2023-03-27T10:22:03.725 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:03 smithi171 ceph-mon[94808]: Deploying daemon mds.k on smithi161 2023-03-27T10:22:03.791 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.791+0000 7fa730944700 1 -- 172.21.15.159:0/361447620 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa72c061660 con 0x7fa72c100810 2023-03-27T10:22:03.791 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.792+0000 7fa71b7fe700 1 -- 172.21.15.159:0/361447620 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 2 v2) v1 ==== 93+0+1260 (secure 0 0 0) 0x7fa724093450 con 0x7fa72c100810 2023-03-27T10:22:03.791 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:03.792 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":2,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24424,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6833/1022778932","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6832","nonce":1022778932},{"type":"v1","addr":"172.21.15.171:6833","nonce":1022778932}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[]} 2023-03-27T10:22:03.793 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.794+0000 7fa730944700 1 -- 172.21.15.159:0/361447620 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa71409b3a0 msgr2=0x7fa71409d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:03.793 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.794+0000 7fa730944700 1 --2- 172.21.15.159:0/361447620 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa71409b3a0 0x7fa71409d850 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fa71c005fd0 tx=0x7fa71c005f60 comp rx=0 tx=0).stop 2023-03-27T10:22:03.794 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.794+0000 7fa730944700 1 -- 172.21.15.159:0/361447620 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa72c100810 msgr2=0x7fa72c19cdd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:03.794 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.794+0000 7fa730944700 1 --2- 172.21.15.159:0/361447620 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa72c100810 0x7fa72c19cdd0 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7fa72400ada0 tx=0x7fa72400c330 comp rx=0 tx=0).stop 2023-03-27T10:22:03.794 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.794+0000 7fa730944700 1 -- 172.21.15.159:0/361447620 shutdown_connections 2023-03-27T10:22:03.794 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.794+0000 7fa730944700 1 --2- 172.21.15.159:0/361447620 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa71409b3a0 0x7fa71409d850 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:03.794 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.794+0000 7fa730944700 1 --2- 172.21.15.159:0/361447620 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa72c0febd0 0x7fa72c068430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:03.794 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.794+0000 7fa730944700 1 --2- 172.21.15.159:0/361447620 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa72c100810 0x7fa72c19cdd0 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:03.794 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.794+0000 7fa730944700 1 --2- 172.21.15.159:0/361447620 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa72c0ffe60 0x7fa72c19a790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:03.795 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.794+0000 7fa730944700 1 -- 172.21.15.159:0/361447620 >> 172.21.15.159:0/361447620 conn(0x7fa72c0fa210 msgr2=0x7fa72c0fc4e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:03.795 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.794+0000 7fa730944700 1 -- 172.21.15.159:0/361447620 shutdown_connections 2023-03-27T10:22:03.795 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:03.794+0000 7fa730944700 1 -- 172.21.15.159:0/361447620 wait complete. 2023-03-27T10:22:03.795 INFO:teuthology.orchestra.run.smithi159.stderr:dumped fsmap epoch 2 2023-03-27T10:22:03.804 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:03 smithi159 ceph-mon[93039]: Deploying daemon mds.k on smithi161 2023-03-27T10:22:04.334 INFO:journalctl@ceph.mds.k.smithi161.stdout:Mar 27 10:22:03 smithi161 ceph-mds[118424]: -- 172.21.15.161:0/367524309 <== mon.2 v2:172.21.15.161:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557850580b60 con 0x557850556800 2023-03-27T10:22:04.334 INFO:journalctl@ceph.mds.k.smithi161.stdout:Mar 27 10:22:03 smithi161 bash[118384]: b2e85083ab7c4266e21bf8e8e16b133f2c803973272fe30318c1a481031df32e 2023-03-27T10:22:04.334 INFO:journalctl@ceph.mds.k.smithi161.stdout:Mar 27 10:22:03 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-k[118400]: starting mds.k at 2023-03-27T10:22:04.334 INFO:journalctl@ceph.mds.k.smithi161.stdout:Mar 27 10:22:03 smithi161 systemd[1]: Started Ceph mds.k for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:22:04.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:04 smithi171 ceph-mon[94808]: pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:04.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:04 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/361447620' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:04.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:04 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:04.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:04 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:04.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:04 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:04.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:04 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:04.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:04 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:04.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:04 smithi161 ceph-mon[94333]: pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:04.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:04 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/361447620' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:04.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:04 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:04.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:04 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:04.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:04 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:04.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:04 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:04.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:04 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:04.853 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:04 smithi159 ceph-mon[93039]: pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:04.853 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:04 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/361447620' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:04.853 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:04 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:04.853 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:04 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:04.854 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:04 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:04.854 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:04 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:04.854 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:04 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:05.479 INFO:journalctl@ceph.mds.j.smithi159.stdout:Mar 27 10:22:05 smithi159 systemd[1]: Starting Ceph mds.j for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:05.480 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:05 smithi159 ceph-mon[93039]: Deploying daemon mds.j on smithi159 2023-03-27T10:22:05.480 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:05 smithi159 ceph-mon[93039]: mds.? [v2:172.21.15.161:6832/2969583384,v1:172.21.15.161:6833/2969583384] up:boot 2023-03-27T10:22:05.480 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:05 smithi159 ceph-mon[93039]: fsmap 2 up:standby 2023-03-27T10:22:05.480 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:05 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2023-03-27T10:22:05.806 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-03-27T10:22:05.829 INFO:journalctl@ceph.mds.j.smithi159.stdout:Mar 27 10:22:05 smithi159 podman[126242]: 2023-03-27T10:22:05.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:05 smithi161 ceph-mon[94333]: Deploying daemon mds.j on smithi159 2023-03-27T10:22:05.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:05 smithi161 ceph-mon[94333]: mds.? [v2:172.21.15.161:6832/2969583384,v1:172.21.15.161:6833/2969583384] up:boot 2023-03-27T10:22:05.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:05 smithi161 ceph-mon[94333]: fsmap 2 up:standby 2023-03-27T10:22:05.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:05 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2023-03-27T10:22:05.936 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.936+0000 7ff4c93b5700 1 -- 172.21.15.159:0/816275542 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff4c40febd0 msgr2=0x7ff4c40fefe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:05.936 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.936+0000 7ff4c93b5700 1 --2- 172.21.15.159:0/816275542 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff4c40febd0 0x7ff4c40fefe0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7ff4bc00b0a0 tx=0x7ff4bc00b3b0 comp rx=0 tx=0).stop 2023-03-27T10:22:05.936 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.936+0000 7ff4c93b5700 1 -- 172.21.15.159:0/816275542 shutdown_connections 2023-03-27T10:22:05.936 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.936+0000 7ff4c93b5700 1 --2- 172.21.15.159:0/816275542 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7ff4c4100810 0x7ff4c41070b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:05.937 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.936+0000 7ff4c93b5700 1 --2- 172.21.15.159:0/816275542 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff4c40febd0 0x7ff4c40fefe0 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:05.937 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.936+0000 7ff4c93b5700 1 --2- 172.21.15.159:0/816275542 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7ff4c40ffe60 0x7ff4c41002d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:05.937 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.936+0000 7ff4c93b5700 1 -- 172.21.15.159:0/816275542 >> 172.21.15.159:0/816275542 conn(0x7ff4c40fa210 msgr2=0x7ff4c40fc620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:05.937 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.936+0000 7ff4c93b5700 1 -- 172.21.15.159:0/816275542 shutdown_connections 2023-03-27T10:22:05.937 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.937+0000 7ff4c93b5700 1 -- 172.21.15.159:0/816275542 wait complete. 2023-03-27T10:22:05.939 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.937+0000 7ff4c93b5700 1 Processor -- start 2023-03-27T10:22:05.939 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.937+0000 7ff4c93b5700 1 -- start start 2023-03-27T10:22:05.939 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c93b5700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7ff4c40febd0 0x7ff4c4195be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:05.939 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c93b5700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff4c40ffe60 0x7ff4c4196120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:05.939 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c93b5700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7ff4c4100810 0x7ff4c419d190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:05.940 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c93b5700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4c419d760 con 0x7ff4c40ffe60 2023-03-27T10:22:05.940 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c93b5700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4c419d8a0 con 0x7ff4c4100810 2023-03-27T10:22:05.940 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c93b5700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4c419da30 con 0x7ff4c40febd0 2023-03-27T10:22:05.940 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c27fc700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff4c40ffe60 0x7ff4c4196120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:05.940 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c27fc700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff4c40ffe60 0x7ff4c4196120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45368/0 (socket says 172.21.15.159:45368) 2023-03-27T10:22:05.940 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c27fc700 1 -- 172.21.15.159:0/2913667503 learned_addr learned my addr 172.21.15.159:0/2913667503 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:05.940 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c2ffd700 1 --2- 172.21.15.159:0/2913667503 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7ff4c40febd0 0x7ff4c4195be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:05.940 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c37fe700 1 --2- 172.21.15.159:0/2913667503 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7ff4c4100810 0x7ff4c419d190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:05.941 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c27fc700 1 -- 172.21.15.159:0/2913667503 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7ff4c40febd0 msgr2=0x7ff4c4195be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:05.941 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c27fc700 1 --2- 172.21.15.159:0/2913667503 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7ff4c40febd0 0x7ff4c4195be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:05.941 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c27fc700 1 -- 172.21.15.159:0/2913667503 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7ff4c4100810 msgr2=0x7ff4c419d190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:05.941 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c27fc700 1 --2- 172.21.15.159:0/2913667503 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7ff4c4100810 0x7ff4c419d190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:05.941 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c27fc700 1 -- 172.21.15.159:0/2913667503 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4bc009d00 con 0x7ff4c40ffe60 2023-03-27T10:22:05.941 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c37fe700 1 --2- 172.21.15.159:0/2913667503 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7ff4c4100810 0x7ff4c419d190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-03-27T10:22:05.941 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c27fc700 1 --2- 172.21.15.159:0/2913667503 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff4c40ffe60 0x7ff4c4196120 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7ff4b800d980 tx=0x7ff4b800dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:05.942 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4abfff700 1 -- 172.21.15.159:0/2913667503 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4b8004220 con 0x7ff4c40ffe60 2023-03-27T10:22:05.942 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c93b5700 1 -- 172.21.15.159:0/2913667503 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff4c410a530 con 0x7ff4c40ffe60 2023-03-27T10:22:05.942 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4abfff700 1 -- 172.21.15.159:0/2913667503 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff4b8007640 con 0x7ff4c40ffe60 2023-03-27T10:22:05.942 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4abfff700 1 -- 172.21.15.159:0/2913667503 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4b8005610 con 0x7ff4c40ffe60 2023-03-27T10:22:05.942 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.938+0000 7ff4c93b5700 1 -- 172.21.15.159:0/2913667503 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff4c410a3a0 con 0x7ff4c40ffe60 2023-03-27T10:22:05.942 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.939+0000 7ff4c93b5700 1 -- 172.21.15.159:0/2913667503 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff4c419e0e0 con 0x7ff4c40ffe60 2023-03-27T10:22:05.943 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.940+0000 7ff4abfff700 1 -- 172.21.15.159:0/2913667503 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7ff4b8021070 con 0x7ff4c40ffe60 2023-03-27T10:22:05.943 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.941+0000 7ff4abfff700 1 --2- 172.21.15.159:0/2913667503 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7ff4ac09b420 0x7ff4ac09d8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:05.943 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.941+0000 7ff4c2ffd700 1 --2- 172.21.15.159:0/2913667503 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7ff4ac09b420 0x7ff4ac09d8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:05.943 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.941+0000 7ff4abfff700 1 -- 172.21.15.159:0/2913667503 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7ff4b80c4930 con 0x7ff4c40ffe60 2023-03-27T10:22:05.944 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.941+0000 7ff4c2ffd700 1 --2- 172.21.15.159:0/2913667503 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7ff4ac09b420 0x7ff4ac09d8d0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7ff4bc007a60 tx=0x7ff4bc00b700 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:05.944 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:05.945+0000 7ff4abfff700 1 -- 172.21.15.159:0/2913667503 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7ff4b8090650 con 0x7ff4c40ffe60 2023-03-27T10:22:05.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:05 smithi171 ceph-mon[94808]: Deploying daemon mds.j on smithi159 2023-03-27T10:22:05.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:05 smithi171 ceph-mon[94808]: mds.? [v2:172.21.15.161:6832/2969583384,v1:172.21.15.161:6833/2969583384] up:boot 2023-03-27T10:22:05.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:05 smithi171 ceph-mon[94808]: fsmap 2 up:standby 2023-03-27T10:22:05.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:05 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2023-03-27T10:22:06.100 INFO:journalctl@ceph.mds.j.smithi159.stdout:Mar 27 10:22:05 smithi159 ceph-mds[126326]: -- 172.21.15.159:0/2409444892 <== mon.0 v2:172.21.15.159:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55732c5f6b60 con 0x55732c5cd000 2023-03-27T10:22:06.100 INFO:journalctl@ceph.mds.j.smithi159.stdout:Mar 27 10:22:05 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-j[126258]: starting mds.j at 2023-03-27T10:22:06.100 INFO:journalctl@ceph.mds.j.smithi159.stdout:Mar 27 10:22:06 smithi159 bash[126242]: a8d923118d8cbd34694425b592c29cef7ac9c63e134df9ea447f2bfb920c522d 2023-03-27T10:22:06.193 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:06.194+0000 7ff4c93b5700 1 -- 172.21.15.159:0/2913667503 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff4c419e690 con 0x7ff4c40ffe60 2023-03-27T10:22:06.194 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:06.194+0000 7ff4abfff700 1 -- 172.21.15.159:0/2913667503 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 3 v3) v1 ==== 93+0+2020 (secure 0 0 0) 0x7ff4b8016090 con 0x7ff4c40ffe60 2023-03-27T10:22:06.194 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:06.194 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":3,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24424,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6833/1022778932","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6832","nonce":1022778932},{"type":"v1","addr":"172.21.15.171:6833","nonce":1022778932}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24440,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6833/2969583384","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6832","nonce":2969583384},{"type":"v1","addr":"172.21.15.161:6833","nonce":2969583384}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[]} 2023-03-27T10:22:06.196 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:06.197+0000 7ff4c93b5700 1 -- 172.21.15.159:0/2913667503 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7ff4ac09b420 msgr2=0x7ff4ac09d8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:06.196 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:06.197+0000 7ff4c93b5700 1 --2- 172.21.15.159:0/2913667503 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7ff4ac09b420 0x7ff4ac09d8d0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7ff4bc007a60 tx=0x7ff4bc00b700 comp rx=0 tx=0).stop 2023-03-27T10:22:06.196 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:06.197+0000 7ff4c93b5700 1 -- 172.21.15.159:0/2913667503 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff4c40ffe60 msgr2=0x7ff4c4196120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:06.197 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:06.197+0000 7ff4c93b5700 1 --2- 172.21.15.159:0/2913667503 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff4c40ffe60 0x7ff4c4196120 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7ff4b800d980 tx=0x7ff4b800dc90 comp rx=0 tx=0).stop 2023-03-27T10:22:06.197 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:06.197+0000 7ff4c93b5700 1 -- 172.21.15.159:0/2913667503 shutdown_connections 2023-03-27T10:22:06.197 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:06.197+0000 7ff4c93b5700 1 --2- 172.21.15.159:0/2913667503 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7ff4ac09b420 0x7ff4ac09d8d0 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:06.197 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:06.197+0000 7ff4c93b5700 1 --2- 172.21.15.159:0/2913667503 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7ff4c40febd0 0x7ff4c4195be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:06.197 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:06.197+0000 7ff4c93b5700 1 --2- 172.21.15.159:0/2913667503 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7ff4c4100810 0x7ff4c419d190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:06.197 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:06.197+0000 7ff4c93b5700 1 --2- 172.21.15.159:0/2913667503 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff4c40ffe60 0x7ff4c4196120 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:06.197 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:06.197+0000 7ff4c93b5700 1 -- 172.21.15.159:0/2913667503 >> 172.21.15.159:0/2913667503 conn(0x7ff4c40fa210 msgr2=0x7ff4c40678e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:06.198 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:06.197+0000 7ff4c93b5700 1 -- 172.21.15.159:0/2913667503 shutdown_connections 2023-03-27T10:22:06.198 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:06.197+0000 7ff4c93b5700 1 -- 172.21.15.159:0/2913667503 wait complete. 2023-03-27T10:22:06.198 INFO:teuthology.orchestra.run.smithi159.stderr:dumped fsmap epoch 3 2023-03-27T10:22:06.364 INFO:journalctl@ceph.mds.j.smithi159.stdout:Mar 27 10:22:06 smithi159 systemd[1]: Started Ceph mds.j for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:22:06.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:06 smithi161 ceph-mon[94333]: pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:06.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:06.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:06.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:06.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:06.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:06 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:06.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:06 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/2913667503' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:06.851 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:06 smithi159 ceph-mon[93039]: pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:06.851 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:06.852 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:06.852 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:06.852 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:06.852 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:06 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:06.852 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:06 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/2913667503' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:06.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:06 smithi171 ceph-mon[94808]: pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:06.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:06.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:06.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:06.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:06.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:06 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:06.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:06 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/2913667503' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:07.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:07 smithi161 ceph-mon[94333]: Deploying daemon mds.d on smithi159 2023-03-27T10:22:07.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:07 smithi161 ceph-mon[94333]: mds.? [v2:172.21.15.159:6834/3048624466,v1:172.21.15.159:6835/3048624466] up:boot 2023-03-27T10:22:07.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:07 smithi161 ceph-mon[94333]: fsmap 3 up:standby 2023-03-27T10:22:07.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:07 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2023-03-27T10:22:07.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:07 smithi159 ceph-mon[93039]: Deploying daemon mds.d on smithi159 2023-03-27T10:22:07.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:07 smithi159 ceph-mon[93039]: mds.? [v2:172.21.15.159:6834/3048624466,v1:172.21.15.159:6835/3048624466] up:boot 2023-03-27T10:22:07.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:07 smithi159 ceph-mon[93039]: fsmap 3 up:standby 2023-03-27T10:22:07.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:07 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2023-03-27T10:22:07.866 INFO:journalctl@ceph.mds.d.smithi159.stdout:Mar 27 10:22:07 smithi159 systemd[1]: Starting Ceph mds.d for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:07.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:07 smithi171 ceph-mon[94808]: Deploying daemon mds.d on smithi159 2023-03-27T10:22:07.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:07 smithi171 ceph-mon[94808]: mds.? [v2:172.21.15.159:6834/3048624466,v1:172.21.15.159:6835/3048624466] up:boot 2023-03-27T10:22:07.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:07 smithi171 ceph-mon[94808]: fsmap 3 up:standby 2023-03-27T10:22:07.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:07 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2023-03-27T10:22:08.210 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-03-27T10:22:08.234 INFO:journalctl@ceph.mds.d.smithi159.stdout:Mar 27 10:22:07 smithi159 podman[126628]: 2023-03-27T10:22:08.343 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.344+0000 7f3449e3d700 1 -- 172.21.15.159:0/219665220 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f34440ffe00 msgr2=0x7f3444100250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:08.343 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.344+0000 7f3449e3d700 1 --2- 172.21.15.159:0/219665220 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f34440ffe00 0x7f3444100250 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f3438009990 tx=0x7f3438009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:08.343 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.344+0000 7f3449e3d700 1 -- 172.21.15.159:0/219665220 shutdown_connections 2023-03-27T10:22:08.343 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.344+0000 7f3449e3d700 1 --2- 172.21.15.159:0/219665220 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f3444100790 0x7f3444107050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:08.344 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.344+0000 7f3449e3d700 1 --2- 172.21.15.159:0/219665220 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f34440fec10 0x7f34440ff020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:08.344 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.344+0000 7f3449e3d700 1 --2- 172.21.15.159:0/219665220 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f34440ffe00 0x7f3444100250 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:08.344 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.344+0000 7f3449e3d700 1 -- 172.21.15.159:0/219665220 >> 172.21.15.159:0/219665220 conn(0x7f34440fa1f0 msgr2=0x7f34440fc660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:08.344 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.344+0000 7f3449e3d700 1 -- 172.21.15.159:0/219665220 shutdown_connections 2023-03-27T10:22:08.344 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.344+0000 7f3449e3d700 1 -- 172.21.15.159:0/219665220 wait complete. 2023-03-27T10:22:08.346 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f3449e3d700 1 Processor -- start 2023-03-27T10:22:08.346 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f3449e3d700 1 -- start start 2023-03-27T10:22:08.346 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f3449e3d700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f34440fec10 0x7f3444194970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:08.346 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f3449e3d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f34440ffe00 0x7f3444194eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:08.347 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f3449e3d700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f3444100790 0x7f344419bf20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:08.347 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f3449e3d700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f344419c4f0 con 0x7f34440ffe00 2023-03-27T10:22:08.347 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f3449e3d700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f344419c630 con 0x7f3444100790 2023-03-27T10:22:08.347 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f3449e3d700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f344419c7c0 con 0x7f34440fec10 2023-03-27T10:22:08.347 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f3442ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f34440ffe00 0x7f3444194eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:08.347 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f3442ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f34440ffe00 0x7f3444194eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45406/0 (socket says 172.21.15.159:45406) 2023-03-27T10:22:08.347 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f3442ffd700 1 -- 172.21.15.159:0/3339586710 learned_addr learned my addr 172.21.15.159:0/3339586710 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:08.347 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f34437fe700 1 --2- 172.21.15.159:0/3339586710 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f34440fec10 0x7f3444194970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:08.348 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f3443fff700 1 --2- 172.21.15.159:0/3339586710 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f3444100790 0x7f344419bf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:08.348 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f3442ffd700 1 -- 172.21.15.159:0/3339586710 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f34440fec10 msgr2=0x7f3444194970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:08.348 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f3442ffd700 1 --2- 172.21.15.159:0/3339586710 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f34440fec10 0x7f3444194970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:08.348 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f3442ffd700 1 -- 172.21.15.159:0/3339586710 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f3444100790 msgr2=0x7f344419bf20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:08.348 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.346+0000 7f3442ffd700 1 --2- 172.21.15.159:0/3339586710 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f3444100790 0x7f344419bf20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:08.348 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.347+0000 7f3442ffd700 1 -- 172.21.15.159:0/3339586710 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f34380095f0 con 0x7f34440ffe00 2023-03-27T10:22:08.348 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.347+0000 7f34437fe700 1 --2- 172.21.15.159:0/3339586710 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f34440fec10 0x7f3444194970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:22:08.348 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.347+0000 7f3442ffd700 1 --2- 172.21.15.159:0/3339586710 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f34440ffe00 0x7f3444194eb0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f3438000c00 tx=0x7f34380047a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:08.349 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.347+0000 7f3440ff9700 1 -- 172.21.15.159:0/3339586710 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f343801d070 con 0x7f34440ffe00 2023-03-27T10:22:08.349 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.347+0000 7f3440ff9700 1 -- 172.21.15.159:0/3339586710 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3438022470 con 0x7f34440ffe00 2023-03-27T10:22:08.349 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.347+0000 7f3449e3d700 1 -- 172.21.15.159:0/3339586710 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f344410a570 con 0x7f34440ffe00 2023-03-27T10:22:08.349 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.347+0000 7f3440ff9700 1 -- 172.21.15.159:0/3339586710 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3438017650 con 0x7f34440ffe00 2023-03-27T10:22:08.349 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.347+0000 7f3449e3d700 1 -- 172.21.15.159:0/3339586710 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f344410a3e0 con 0x7f34440ffe00 2023-03-27T10:22:08.350 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.348+0000 7f3449e3d700 1 -- 172.21.15.159:0/3339586710 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f34441061d0 con 0x7f34440ffe00 2023-03-27T10:22:08.350 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.349+0000 7f3440ff9700 1 -- 172.21.15.159:0/3339586710 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f3438029030 con 0x7f34440ffe00 2023-03-27T10:22:08.350 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.350+0000 7f3440ff9700 1 --2- 172.21.15.159:0/3339586710 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f342c09b3a0 0x7f342c09d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:08.350 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.350+0000 7f34437fe700 1 --2- 172.21.15.159:0/3339586710 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f342c09b3a0 0x7f342c09d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:08.350 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.350+0000 7f3440ff9700 1 -- 172.21.15.159:0/3339586710 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f34380c5f80 con 0x7f34440ffe00 2023-03-27T10:22:08.351 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.351+0000 7f34437fe700 1 --2- 172.21.15.159:0/3339586710 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f342c09b3a0 0x7f342c09d850 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f34440ffc60 tx=0x7f3434009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:08.353 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.355+0000 7f3440ff9700 1 -- 172.21.15.159:0/3339586710 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f3438091e20 con 0x7f34440ffe00 2023-03-27T10:22:08.497 INFO:journalctl@ceph.mds.d.smithi159.stdout:Mar 27 10:22:08 smithi159 bash[126628]: 2729730eb4330008c7d01192164a1ec25a47c63c9f3803d78b2425c8acdbcd23 2023-03-27T10:22:08.497 INFO:journalctl@ceph.mds.d.smithi159.stdout:Mar 27 10:22:08 smithi159 ceph-mds[126668]: -- 172.21.15.159:0/2891076485 <== mon.0 v2:172.21.15.159:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558eae936b60 con 0x558eae90cc00 2023-03-27T10:22:08.497 INFO:journalctl@ceph.mds.d.smithi159.stdout:Mar 27 10:22:08 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-d[126644]: starting mds.d at 2023-03-27T10:22:08.498 INFO:journalctl@ceph.mds.d.smithi159.stdout:Mar 27 10:22:08 smithi159 systemd[1]: Started Ceph mds.d for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:22:08.586 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.588+0000 7f3449e3d700 1 -- 172.21.15.159:0/3339586710 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3444061860 con 0x7f34440ffe00 2023-03-27T10:22:08.587 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.588+0000 7f3440ff9700 1 -- 172.21.15.159:0/3339586710 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 5 v5) v1 ==== 93+0+3540 (secure 0 0 0) 0x7f3438095440 con 0x7f34440ffe00 2023-03-27T10:22:08.587 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:08.587 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":5,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14556,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6835/3048624466","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6834","nonce":3048624466},{"type":"v1","addr":"172.21.15.159:6835","nonce":3048624466}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14562,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6837/1998956589","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6836","nonce":1998956589},{"type":"v1","addr":"172.21.15.159:6837","nonce":1998956589}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24424,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6833/1022778932","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6832","nonce":1022778932},{"type":"v1","addr":"172.21.15.171:6833","nonce":1022778932}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24440,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6833/2969583384","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6832","nonce":2969583384},{"type":"v1","addr":"172.21.15.161:6833","nonce":2969583384}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[]} 2023-03-27T10:22:08.589 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.590+0000 7f3449e3d700 1 -- 172.21.15.159:0/3339586710 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f342c09b3a0 msgr2=0x7f342c09d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:08.589 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.590+0000 7f3449e3d700 1 --2- 172.21.15.159:0/3339586710 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f342c09b3a0 0x7f342c09d850 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f34440ffc60 tx=0x7f3434009260 comp rx=0 tx=0).stop 2023-03-27T10:22:08.589 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.590+0000 7f3449e3d700 1 -- 172.21.15.159:0/3339586710 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f34440ffe00 msgr2=0x7f3444194eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:08.589 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.590+0000 7f3449e3d700 1 --2- 172.21.15.159:0/3339586710 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f34440ffe00 0x7f3444194eb0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f3438000c00 tx=0x7f34380047a0 comp rx=0 tx=0).stop 2023-03-27T10:22:08.590 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.591+0000 7f3449e3d700 1 -- 172.21.15.159:0/3339586710 shutdown_connections 2023-03-27T10:22:08.590 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.591+0000 7f3449e3d700 1 --2- 172.21.15.159:0/3339586710 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f342c09b3a0 0x7f342c09d850 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:08.590 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.591+0000 7f3449e3d700 1 --2- 172.21.15.159:0/3339586710 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f34440fec10 0x7f3444194970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:08.590 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.591+0000 7f3449e3d700 1 --2- 172.21.15.159:0/3339586710 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f3444100790 0x7f344419bf20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:08.590 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.591+0000 7f3449e3d700 1 --2- 172.21.15.159:0/3339586710 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f34440ffe00 0x7f3444194eb0 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:08.590 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.591+0000 7f3449e3d700 1 -- 172.21.15.159:0/3339586710 >> 172.21.15.159:0/3339586710 conn(0x7f34440fa1f0 msgr2=0x7f3444103310 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:08.590 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.591+0000 7f3449e3d700 1 -- 172.21.15.159:0/3339586710 shutdown_connections 2023-03-27T10:22:08.590 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:08.591+0000 7f3449e3d700 1 -- 172.21.15.159:0/3339586710 wait complete. 2023-03-27T10:22:08.591 INFO:teuthology.orchestra.run.smithi159.stderr:dumped fsmap epoch 5 2023-03-27T10:22:08.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:08 smithi161 ceph-mon[94333]: pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:08.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:08 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:08.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:08 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:08.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:08 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:08.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:08 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:08.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:08 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:08.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:08 smithi159 ceph-mon[93039]: pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:08.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:08.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:08.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:08.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:08.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:08 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:08.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:08 smithi171 ceph-mon[94808]: pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:08.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:08.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:08.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:08.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:08.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:08 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:09.794 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:09 smithi161 ceph-mon[94333]: Deploying daemon mds.e on smithi161 2023-03-27T10:22:09.794 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:09 smithi161 ceph-mon[94333]: mds.? [v2:172.21.15.159:6836/1998956589,v1:172.21.15.159:6837/1998956589] up:boot 2023-03-27T10:22:09.794 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:09 smithi161 ceph-mon[94333]: fsmap 4 up:standby 2023-03-27T10:22:09.794 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:09 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2023-03-27T10:22:09.794 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:09 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3339586710' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:09.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:09 smithi159 ceph-mon[93039]: Deploying daemon mds.e on smithi161 2023-03-27T10:22:09.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:09 smithi159 ceph-mon[93039]: mds.? [v2:172.21.15.159:6836/1998956589,v1:172.21.15.159:6837/1998956589] up:boot 2023-03-27T10:22:09.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:09 smithi159 ceph-mon[93039]: fsmap 4 up:standby 2023-03-27T10:22:09.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:09 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2023-03-27T10:22:09.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:09 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3339586710' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:09.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:09 smithi171 ceph-mon[94808]: Deploying daemon mds.e on smithi161 2023-03-27T10:22:09.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:09 smithi171 ceph-mon[94808]: mds.? [v2:172.21.15.159:6836/1998956589,v1:172.21.15.159:6837/1998956589] up:boot 2023-03-27T10:22:09.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:09 smithi171 ceph-mon[94808]: fsmap 4 up:standby 2023-03-27T10:22:09.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:09 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2023-03-27T10:22:09.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:09 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3339586710' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:10.084 INFO:journalctl@ceph.mds.e.smithi161.stdout:Mar 27 10:22:09 smithi161 systemd[1]: Starting Ceph mds.e for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:10.454 INFO:journalctl@ceph.mds.e.smithi161.stdout:Mar 27 10:22:10 smithi161 podman[118733]: 2023-03-27T10:22:10.602 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-03-27T10:22:10.735 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.736+0000 7ff863875700 1 -- 172.21.15.159:0/705407828 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff85c100810 msgr2=0x7ff85c1070b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:10.735 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.736+0000 7ff863875700 1 --2- 172.21.15.159:0/705407828 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff85c100810 0x7ff85c1070b0 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7ff85400b0a0 tx=0x7ff85400b3b0 comp rx=0 tx=0).stop 2023-03-27T10:22:10.735 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.736+0000 7ff863875700 1 -- 172.21.15.159:0/705407828 shutdown_connections 2023-03-27T10:22:10.736 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.736+0000 7ff863875700 1 --2- 172.21.15.159:0/705407828 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff85c100810 0x7ff85c1070b0 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:10.736 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.736+0000 7ff863875700 1 --2- 172.21.15.159:0/705407828 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7ff85c0febd0 0x7ff85c0fefe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:10.736 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.736+0000 7ff863875700 1 --2- 172.21.15.159:0/705407828 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7ff85c0ffe60 0x7ff85c1002d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:10.736 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.736+0000 7ff863875700 1 -- 172.21.15.159:0/705407828 >> 172.21.15.159:0/705407828 conn(0x7ff85c0fa210 msgr2=0x7ff85c0fc620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:10.736 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.737+0000 7ff863875700 1 -- 172.21.15.159:0/705407828 shutdown_connections 2023-03-27T10:22:10.736 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.737+0000 7ff863875700 1 -- 172.21.15.159:0/705407828 wait complete. 2023-03-27T10:22:10.738 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.737+0000 7ff863875700 1 Processor -- start 2023-03-27T10:22:10.738 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.737+0000 7ff863875700 1 -- start start 2023-03-27T10:22:10.738 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.737+0000 7ff863875700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7ff85c0febd0 0x7ff85c195b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:10.739 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.737+0000 7ff863875700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7ff85c0ffe60 0x7ff85c1960a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:10.739 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.737+0000 7ff863875700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff85c100810 0x7ff85c19d110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:10.739 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.737+0000 7ff863875700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff85c19d6e0 con 0x7ff85c100810 2023-03-27T10:22:10.739 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.737+0000 7ff863875700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff85c19d820 con 0x7ff85c0ffe60 2023-03-27T10:22:10.739 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.737+0000 7ff863875700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff85c19d9b0 con 0x7ff85c0febd0 2023-03-27T10:22:10.739 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.737+0000 7ff861e12700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff85c100810 0x7ff85c19d110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:10.739 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.737+0000 7ff861611700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7ff85c0febd0 0x7ff85c195b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:10.739 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.737+0000 7ff861e12700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff85c100810 0x7ff85c19d110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45424/0 (socket says 172.21.15.159:45424) 2023-03-27T10:22:10.740 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.737+0000 7ff861611700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7ff85c0febd0 0x7ff85c195b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.171:3300/0 says I am v2:172.21.15.159:45958/0 (socket says 172.21.15.159:45958) 2023-03-27T10:22:10.740 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.737+0000 7ff861e12700 1 -- 172.21.15.159:0/1236682603 learned_addr learned my addr 172.21.15.159:0/1236682603 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:10.740 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.738+0000 7ff860e10700 1 --2- 172.21.15.159:0/1236682603 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7ff85c0ffe60 0x7ff85c1960a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:10.740 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.738+0000 7ff861e12700 1 -- 172.21.15.159:0/1236682603 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7ff85c0febd0 msgr2=0x7ff85c195b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:10.740 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.738+0000 7ff861e12700 1 --2- 172.21.15.159:0/1236682603 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7ff85c0febd0 0x7ff85c195b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:10.740 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.738+0000 7ff861e12700 1 -- 172.21.15.159:0/1236682603 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7ff85c0ffe60 msgr2=0x7ff85c1960a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:10.740 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.738+0000 7ff861e12700 1 --2- 172.21.15.159:0/1236682603 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7ff85c0ffe60 0x7ff85c1960a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:10.741 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.738+0000 7ff861e12700 1 -- 172.21.15.159:0/1236682603 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff854009d00 con 0x7ff85c100810 2023-03-27T10:22:10.741 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.738+0000 7ff860e10700 1 --2- 172.21.15.159:0/1236682603 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7ff85c0ffe60 0x7ff85c1960a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-03-27T10:22:10.741 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.738+0000 7ff861e12700 1 --2- 172.21.15.159:0/1236682603 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff85c100810 0x7ff85c19d110 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7ff85400b780 tx=0x7ff85400bab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:10.741 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.738+0000 7ff8527fc700 1 -- 172.21.15.159:0/1236682603 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff85400e040 con 0x7ff85c100810 2023-03-27T10:22:10.741 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.738+0000 7ff8527fc700 1 -- 172.21.15.159:0/1236682603 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff854004c50 con 0x7ff85c100810 2023-03-27T10:22:10.741 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.738+0000 7ff863875700 1 -- 172.21.15.159:0/1236682603 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff85c10a530 con 0x7ff85c100810 2023-03-27T10:22:10.741 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.738+0000 7ff8527fc700 1 -- 172.21.15.159:0/1236682603 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8540053b0 con 0x7ff85c100810 2023-03-27T10:22:10.741 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.738+0000 7ff863875700 1 -- 172.21.15.159:0/1236682603 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff85c10a3a0 con 0x7ff85c100810 2023-03-27T10:22:10.742 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.739+0000 7ff863875700 1 -- 172.21.15.159:0/1236682603 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff85c0674c0 con 0x7ff85c100810 2023-03-27T10:22:10.742 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.740+0000 7ff8527fc700 1 -- 172.21.15.159:0/1236682603 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7ff85402d070 con 0x7ff85c100810 2023-03-27T10:22:10.742 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.741+0000 7ff8527fc700 1 --2- 172.21.15.159:0/1236682603 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7ff84809b3a0 0x7ff84809d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:10.743 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.741+0000 7ff861611700 1 --2- 172.21.15.159:0/1236682603 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7ff84809b3a0 0x7ff84809d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:10.743 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.741+0000 7ff8527fc700 1 -- 172.21.15.159:0/1236682603 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7ff8540c6450 con 0x7ff85c100810 2023-03-27T10:22:10.743 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.741+0000 7ff861611700 1 --2- 172.21.15.159:0/1236682603 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7ff84809b3a0 0x7ff84809d850 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7ff858009c60 tx=0x7ff858009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:10.743 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.744+0000 7ff8527fc700 1 -- 172.21.15.159:0/1236682603 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7ff8540922f0 con 0x7ff85c100810 2023-03-27T10:22:10.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:10 smithi161 ceph-mon[94333]: pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:10.834 INFO:journalctl@ceph.mds.e.smithi161.stdout:Mar 27 10:22:10 smithi161 bash[118733]: a18f976e612caffffd10508eb7d65833fefe904eb441314604802bb965749cf0 2023-03-27T10:22:10.834 INFO:journalctl@ceph.mds.e.smithi161.stdout:Mar 27 10:22:10 smithi161 ceph-mds[118774]: -- 172.21.15.161:0/1283962828 <== mon.2 v2:172.21.15.161:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557e72e6cb60 con 0x557e72e42c00 2023-03-27T10:22:10.834 INFO:journalctl@ceph.mds.e.smithi161.stdout:Mar 27 10:22:10 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-e[118749]: starting mds.e at 2023-03-27T10:22:10.835 INFO:journalctl@ceph.mds.e.smithi161.stdout:Mar 27 10:22:10 smithi161 systemd[1]: Started Ceph mds.e for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:22:10.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:10 smithi159 ceph-mon[93039]: pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:10.970 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.972+0000 7ff863875700 1 -- 172.21.15.159:0/1236682603 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff85c19e320 con 0x7ff85c100810 2023-03-27T10:22:10.971 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.973+0000 7ff8527fc700 1 -- 172.21.15.159:0/1236682603 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 5 v5) v1 ==== 93+0+3540 (secure 0 0 0) 0x7ff854021df0 con 0x7ff85c100810 2023-03-27T10:22:10.971 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:10.972 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":5,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14556,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6835/3048624466","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6834","nonce":3048624466},{"type":"v1","addr":"172.21.15.159:6835","nonce":3048624466}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14562,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6837/1998956589","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6836","nonce":1998956589},{"type":"v1","addr":"172.21.15.159:6837","nonce":1998956589}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24424,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6833/1022778932","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6832","nonce":1022778932},{"type":"v1","addr":"172.21.15.171:6833","nonce":1022778932}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24440,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6833/2969583384","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6832","nonce":2969583384},{"type":"v1","addr":"172.21.15.161:6833","nonce":2969583384}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[]} 2023-03-27T10:22:10.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.974+0000 7ff863875700 1 -- 172.21.15.159:0/1236682603 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7ff84809b3a0 msgr2=0x7ff84809d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:10.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.975+0000 7ff863875700 1 --2- 172.21.15.159:0/1236682603 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7ff84809b3a0 0x7ff84809d850 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7ff858009c60 tx=0x7ff858009260 comp rx=0 tx=0).stop 2023-03-27T10:22:10.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.975+0000 7ff863875700 1 -- 172.21.15.159:0/1236682603 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff85c100810 msgr2=0x7ff85c19d110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:10.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.975+0000 7ff863875700 1 --2- 172.21.15.159:0/1236682603 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff85c100810 0x7ff85c19d110 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7ff85400b780 tx=0x7ff85400bab0 comp rx=0 tx=0).stop 2023-03-27T10:22:10.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.975+0000 7ff863875700 1 -- 172.21.15.159:0/1236682603 shutdown_connections 2023-03-27T10:22:10.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.975+0000 7ff863875700 1 --2- 172.21.15.159:0/1236682603 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7ff84809b3a0 0x7ff84809d850 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:10.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.975+0000 7ff863875700 1 --2- 172.21.15.159:0/1236682603 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7ff85c0febd0 0x7ff85c195b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:10.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.975+0000 7ff863875700 1 --2- 172.21.15.159:0/1236682603 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7ff85c100810 0x7ff85c19d110 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:10.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.975+0000 7ff863875700 1 --2- 172.21.15.159:0/1236682603 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7ff85c0ffe60 0x7ff85c1960a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:10.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.975+0000 7ff863875700 1 -- 172.21.15.159:0/1236682603 >> 172.21.15.159:0/1236682603 conn(0x7ff85c0fa210 msgr2=0x7ff85c0678c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:10.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.975+0000 7ff863875700 1 -- 172.21.15.159:0/1236682603 shutdown_connections 2023-03-27T10:22:10.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:10.975+0000 7ff863875700 1 -- 172.21.15.159:0/1236682603 wait complete. 2023-03-27T10:22:10.975 INFO:teuthology.orchestra.run.smithi159.stderr:dumped fsmap epoch 5 2023-03-27T10:22:10.975 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:10 smithi171 ceph-mon[94808]: pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:11.808 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:11 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:11.808 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:11 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:11.808 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:11 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:11.808 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:11 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:11.808 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:11 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:11.809 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:11 smithi171 ceph-mon[94808]: Deploying daemon mds.f on smithi171 2023-03-27T10:22:11.809 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:11 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/1236682603' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:11.809 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:11 smithi171 ceph-mon[94808]: mds.? [v2:172.21.15.161:6834/761787339,v1:172.21.15.161:6835/761787339] up:boot 2023-03-27T10:22:11.809 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:11 smithi171 ceph-mon[94808]: fsmap 5 up:standby 2023-03-27T10:22:11.809 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:11 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2023-03-27T10:22:11.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:11.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:11.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:11.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:11.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:11.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:11 smithi161 ceph-mon[94333]: Deploying daemon mds.f on smithi171 2023-03-27T10:22:11.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:11 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/1236682603' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:11.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:11 smithi161 ceph-mon[94333]: mds.? [v2:172.21.15.161:6834/761787339,v1:172.21.15.161:6835/761787339] up:boot 2023-03-27T10:22:11.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:11 smithi161 ceph-mon[94333]: fsmap 5 up:standby 2023-03-27T10:22:11.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:11 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2023-03-27T10:22:11.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:11 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:11.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:11 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:11.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:11 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:11.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:11 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:11.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:11 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:11.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:11 smithi159 ceph-mon[93039]: Deploying daemon mds.f on smithi171 2023-03-27T10:22:11.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:11 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/1236682603' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:11.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:11 smithi159 ceph-mon[93039]: mds.? [v2:172.21.15.161:6834/761787339,v1:172.21.15.161:6835/761787339] up:boot 2023-03-27T10:22:11.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:11 smithi159 ceph-mon[93039]: fsmap 5 up:standby 2023-03-27T10:22:11.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:11 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2023-03-27T10:22:12.226 INFO:journalctl@ceph.mds.f.smithi171.stdout:Mar 27 10:22:12 smithi171 systemd[1]: Starting Ceph mds.f for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:12.612 INFO:journalctl@ceph.mds.f.smithi171.stdout:Mar 27 10:22:12 smithi171 podman[120214]: 2023-03-27T10:22:12.613 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:12 smithi171 ceph-mon[94808]: pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:12.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:12 smithi161 ceph-mon[94333]: pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:12.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:12 smithi159 ceph-mon[93039]: pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:12.976 INFO:journalctl@ceph.mds.f.smithi171.stdout:Mar 27 10:22:12 smithi171 ceph-mds[120254]: -- 172.21.15.171:0/581468156 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5604eb222b60 con 0x5604eb1f9000 2023-03-27T10:22:12.976 INFO:journalctl@ceph.mds.f.smithi171.stdout:Mar 27 10:22:12 smithi171 bash[120214]: 272e5f98ea7210fa15b322117c5600b0938faea29b91e1a418627bd231355212 2023-03-27T10:22:12.976 INFO:journalctl@ceph.mds.f.smithi171.stdout:Mar 27 10:22:12 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-f[120230]: starting mds.f at 2023-03-27T10:22:12.976 INFO:journalctl@ceph.mds.f.smithi171.stdout:Mar 27 10:22:12 smithi171 systemd[1]: Started Ceph mds.f for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:22:12.986 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-03-27T10:22:13.121 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.122+0000 7f248a8c0700 1 -- 172.21.15.159:0/1114780527 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f24840ff550 msgr2=0x7f24840ff9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:13.121 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.122+0000 7f248a8c0700 1 --2- 172.21.15.159:0/1114780527 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f24840ff550 0x7f24840ff9a0 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f247800aaa0 tx=0x7f247800adb0 comp rx=0 tx=0).stop 2023-03-27T10:22:13.121 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.122+0000 7f248a8c0700 1 -- 172.21.15.159:0/1114780527 shutdown_connections 2023-03-27T10:22:13.121 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.122+0000 7f248a8c0700 1 --2- 172.21.15.159:0/1114780527 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f24840ffee0 0x7f24841067a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:13.121 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.122+0000 7f248a8c0700 1 --2- 172.21.15.159:0/1114780527 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f24840fe360 0x7f24840fe770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:13.122 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.122+0000 7f248a8c0700 1 --2- 172.21.15.159:0/1114780527 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f24840ff550 0x7f24840ff9a0 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:13.122 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.122+0000 7f248a8c0700 1 -- 172.21.15.159:0/1114780527 >> 172.21.15.159:0/1114780527 conn(0x7f24840f9980 msgr2=0x7f24840fbdb0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:13.122 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.122+0000 7f248a8c0700 1 -- 172.21.15.159:0/1114780527 shutdown_connections 2023-03-27T10:22:13.122 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.122+0000 7f248a8c0700 1 -- 172.21.15.159:0/1114780527 wait complete. 2023-03-27T10:22:13.122 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f248a8c0700 1 Processor -- start 2023-03-27T10:22:13.122 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f248a8c0700 1 -- start start 2023-03-27T10:22:13.124 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f248a8c0700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f24840fe360 0x7f24841026a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:13.124 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f248a8c0700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f24840ff550 0x7f2484100ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:13.124 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f248a8c0700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f24840ffee0 0x7f2484101220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:13.124 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f248a8c0700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f24841019c0 con 0x7f24840ff550 2023-03-27T10:22:13.125 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f248a8c0700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2484101b30 con 0x7f24840fe360 2023-03-27T10:22:13.125 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f248a8c0700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f248419faf0 con 0x7f24840ffee0 2023-03-27T10:22:13.125 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f24837fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f24840ff550 0x7f2484100ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:13.125 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f24837fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f24840ff550 0x7f2484100ce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45436/0 (socket says 172.21.15.159:45436) 2023-03-27T10:22:13.125 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f24837fe700 1 -- 172.21.15.159:0/1507768515 learned_addr learned my addr 172.21.15.159:0/1507768515 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:13.125 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f2488e5d700 1 --2- 172.21.15.159:0/1507768515 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f24840ffee0 0x7f2484101220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:13.125 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f2483fff700 1 --2- 172.21.15.159:0/1507768515 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f24840fe360 0x7f24841026a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:13.126 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f24837fe700 1 -- 172.21.15.159:0/1507768515 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f24840ffee0 msgr2=0x7f2484101220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:13.126 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f24837fe700 1 --2- 172.21.15.159:0/1507768515 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f24840ffee0 0x7f2484101220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:13.126 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f24837fe700 1 -- 172.21.15.159:0/1507768515 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f24840fe360 msgr2=0x7f24841026a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:13.126 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f24837fe700 1 --2- 172.21.15.159:0/1507768515 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f24840fe360 0x7f24841026a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:13.126 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f24837fe700 1 -- 172.21.15.159:0/1507768515 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f247800a700 con 0x7f24840ff550 2023-03-27T10:22:13.126 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.123+0000 7f2483fff700 1 --2- 172.21.15.159:0/1507768515 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f24840fe360 0x7f24841026a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:22:13.126 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.124+0000 7f24837fe700 1 --2- 172.21.15.159:0/1507768515 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f24840ff550 0x7f2484100ce0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f2478000c00 tx=0x7f2478003cd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:13.126 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.124+0000 7f24817fa700 1 -- 172.21.15.159:0/1507768515 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f24780051e0 con 0x7f24840ff550 2023-03-27T10:22:13.127 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.124+0000 7f24817fa700 1 -- 172.21.15.159:0/1507768515 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f247800e450 con 0x7f24840ff550 2023-03-27T10:22:13.127 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.124+0000 7f248a8c0700 1 -- 172.21.15.159:0/1507768515 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f24841094f0 con 0x7f24840ff550 2023-03-27T10:22:13.127 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.124+0000 7f24817fa700 1 -- 172.21.15.159:0/1507768515 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2478016680 con 0x7f24840ff550 2023-03-27T10:22:13.127 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.124+0000 7f248a8c0700 1 -- 172.21.15.159:0/1507768515 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2484109360 con 0x7f24840ff550 2023-03-27T10:22:13.127 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.125+0000 7f248a8c0700 1 -- 172.21.15.159:0/1507768515 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f24841a0000 con 0x7f24840ff550 2023-03-27T10:22:13.127 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.126+0000 7f24817fa700 1 -- 172.21.15.159:0/1507768515 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f2478025020 con 0x7f24840ff550 2023-03-27T10:22:13.128 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.126+0000 7f24817fa700 1 --2- 172.21.15.159:0/1507768515 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f246c09b3a0 0x7f246c09d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:13.128 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.127+0000 7f2483fff700 1 --2- 172.21.15.159:0/1507768515 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f246c09b3a0 0x7f246c09d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:13.128 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.127+0000 7f24817fa700 1 -- 172.21.15.159:0/1507768515 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f2478012070 con 0x7f24840ff550 2023-03-27T10:22:13.128 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.127+0000 7f2483fff700 1 --2- 172.21.15.159:0/1507768515 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f246c09b3a0 0x7f246c09d850 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f24840ff3b0 tx=0x7f2474009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:13.129 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.130+0000 7f24817fa700 1 -- 172.21.15.159:0/1507768515 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f2478092980 con 0x7f24840ff550 2023-03-27T10:22:13.356 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.357+0000 7f248a8c0700 1 -- 172.21.15.159:0/1507768515 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f24841a06a0 con 0x7f24840ff550 2023-03-27T10:22:13.356 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.358+0000 7f24817fa700 1 -- 172.21.15.159:0/1507768515 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 6 v6) v1 ==== 93+0+4297 (secure 0 0 0) 0x7f247800dea0 con 0x7f24840ff550 2023-03-27T10:22:13.357 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:13.357 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":6,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14556,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6835/3048624466","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6834","nonce":3048624466},{"type":"v1","addr":"172.21.15.159:6835","nonce":3048624466}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14562,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6837/1998956589","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6836","nonce":1998956589},{"type":"v1","addr":"172.21.15.159:6837","nonce":1998956589}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24424,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6833/1022778932","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6832","nonce":1022778932},{"type":"v1","addr":"172.21.15.171:6833","nonce":1022778932}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24440,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6833/2969583384","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6832","nonce":2969583384},{"type":"v1","addr":"172.21.15.161:6833","nonce":2969583384}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24464,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6835/761787339","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6834","nonce":761787339},{"type":"v1","addr":"172.21.15.161:6835","nonce":761787339}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6}],"filesystems":[]} 2023-03-27T10:22:13.358 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.360+0000 7f248a8c0700 1 -- 172.21.15.159:0/1507768515 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f246c09b3a0 msgr2=0x7f246c09d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:13.359 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.360+0000 7f248a8c0700 1 --2- 172.21.15.159:0/1507768515 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f246c09b3a0 0x7f246c09d850 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f24840ff3b0 tx=0x7f2474009260 comp rx=0 tx=0).stop 2023-03-27T10:22:13.359 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.360+0000 7f248a8c0700 1 -- 172.21.15.159:0/1507768515 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f24840ff550 msgr2=0x7f2484100ce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:13.359 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.360+0000 7f248a8c0700 1 --2- 172.21.15.159:0/1507768515 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f24840ff550 0x7f2484100ce0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f2478000c00 tx=0x7f2478003cd0 comp rx=0 tx=0).stop 2023-03-27T10:22:13.359 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.360+0000 7f248a8c0700 1 -- 172.21.15.159:0/1507768515 shutdown_connections 2023-03-27T10:22:13.359 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.360+0000 7f248a8c0700 1 --2- 172.21.15.159:0/1507768515 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f246c09b3a0 0x7f246c09d850 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:13.359 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.360+0000 7f248a8c0700 1 --2- 172.21.15.159:0/1507768515 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f24840ffee0 0x7f2484101220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:13.359 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.360+0000 7f248a8c0700 1 --2- 172.21.15.159:0/1507768515 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f24840fe360 0x7f24841026a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:13.360 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.360+0000 7f248a8c0700 1 --2- 172.21.15.159:0/1507768515 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f24840ff550 0x7f2484100ce0 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:13.360 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.360+0000 7f248a8c0700 1 -- 172.21.15.159:0/1507768515 >> 172.21.15.159:0/1507768515 conn(0x7f24840f9980 msgr2=0x7f24840fbc40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:13.360 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.360+0000 7f248a8c0700 1 -- 172.21.15.159:0/1507768515 shutdown_connections 2023-03-27T10:22:13.360 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:13.360+0000 7f248a8c0700 1 -- 172.21.15.159:0/1507768515 wait complete. 2023-03-27T10:22:13.360 INFO:teuthology.orchestra.run.smithi159.stderr:dumped fsmap epoch 6 2023-03-27T10:22:13.932 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:13 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:13.932 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:13 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:13.932 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:13 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:13.933 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:13 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:13.933 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:13 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:13.933 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:13 smithi171 ceph-mon[94808]: Deploying daemon mds.c on smithi171 2023-03-27T10:22:13.933 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:13 smithi171 ceph-mon[94808]: pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:13.933 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:13 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/1507768515' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:13.933 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:13 smithi171 ceph-mon[94808]: mds.? [v2:172.21.15.171:6834/1823903399,v1:172.21.15.171:6835/1823903399] up:boot 2023-03-27T10:22:13.933 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:13 smithi171 ceph-mon[94808]: fsmap 6 up:standby 2023-03-27T10:22:13.934 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:13 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2023-03-27T10:22:14.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:13 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:14.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:13 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:14.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:13 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:14.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:13 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:14.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:13 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:14.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:13 smithi161 ceph-mon[94333]: Deploying daemon mds.c on smithi171 2023-03-27T10:22:14.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:13 smithi161 ceph-mon[94333]: pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:14.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:13 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/1507768515' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:14.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:13 smithi161 ceph-mon[94333]: mds.? [v2:172.21.15.171:6834/1823903399,v1:172.21.15.171:6835/1823903399] up:boot 2023-03-27T10:22:14.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:13 smithi161 ceph-mon[94333]: fsmap 6 up:standby 2023-03-27T10:22:14.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:13 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2023-03-27T10:22:14.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:13 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:14.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:13 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:14.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:13 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:14.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:13 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:14.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:13 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:14.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:13 smithi159 ceph-mon[93039]: Deploying daemon mds.c on smithi171 2023-03-27T10:22:14.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:13 smithi159 ceph-mon[93039]: pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:14.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:13 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/1507768515' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:14.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:13 smithi159 ceph-mon[93039]: mds.? [v2:172.21.15.171:6834/1823903399,v1:172.21.15.171:6835/1823903399] up:boot 2023-03-27T10:22:14.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:13 smithi159 ceph-mon[93039]: fsmap 6 up:standby 2023-03-27T10:22:14.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:13 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2023-03-27T10:22:14.226 INFO:journalctl@ceph.mds.c.smithi171.stdout:Mar 27 10:22:14 smithi171 systemd[1]: Starting Ceph mds.c for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:14.712 INFO:journalctl@ceph.mds.c.smithi171.stdout:Mar 27 10:22:14 smithi171 podman[120556]: 2023-03-27T10:22:14.976 INFO:journalctl@ceph.mds.c.smithi171.stdout:Mar 27 10:22:14 smithi171 ceph-mds[120596]: -- 172.21.15.171:0/1369805419 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5633b22c4b60 con 0x5633b229a800 2023-03-27T10:22:14.976 INFO:journalctl@ceph.mds.c.smithi171.stdout:Mar 27 10:22:14 smithi171 bash[120556]: 49cbb5434ffdfd210fcee4a57f81b3dfb94e423c659a8f70515b6afcce97bc36 2023-03-27T10:22:14.976 INFO:journalctl@ceph.mds.c.smithi171.stdout:Mar 27 10:22:14 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-c[120572]: starting mds.c at 2023-03-27T10:22:14.976 INFO:journalctl@ceph.mds.c.smithi171.stdout:Mar 27 10:22:14 smithi171 systemd[1]: Started Ceph mds.c for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:22:15.371 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-03-27T10:22:15.506 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.507+0000 7fd90e76d700 1 -- 172.21.15.159:0/2764767042 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd908100810 msgr2=0x7fd9081070b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:15.507 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.507+0000 7fd90e76d700 1 --2- 172.21.15.159:0/2764767042 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd908100810 0x7fd9081070b0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7fd90000b0a0 tx=0x7fd90000b3b0 comp rx=0 tx=0).stop 2023-03-27T10:22:15.507 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.507+0000 7fd90e76d700 1 -- 172.21.15.159:0/2764767042 shutdown_connections 2023-03-27T10:22:15.507 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.507+0000 7fd90e76d700 1 --2- 172.21.15.159:0/2764767042 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd908100810 0x7fd9081070b0 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:15.507 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.507+0000 7fd90e76d700 1 --2- 172.21.15.159:0/2764767042 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd9080febd0 0x7fd9080fefe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:15.507 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.507+0000 7fd90e76d700 1 --2- 172.21.15.159:0/2764767042 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd9080ffe60 0x7fd9081002d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:15.507 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.507+0000 7fd90e76d700 1 -- 172.21.15.159:0/2764767042 >> 172.21.15.159:0/2764767042 conn(0x7fd9080fa210 msgr2=0x7fd9080fc620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:15.508 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.507+0000 7fd90e76d700 1 -- 172.21.15.159:0/2764767042 shutdown_connections 2023-03-27T10:22:15.508 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.507+0000 7fd90e76d700 1 -- 172.21.15.159:0/2764767042 wait complete. 2023-03-27T10:22:15.510 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90e76d700 1 Processor -- start 2023-03-27T10:22:15.510 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90e76d700 1 -- start start 2023-03-27T10:22:15.510 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90e76d700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd9080febd0 0x7fd908068450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:15.510 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90e76d700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd9080ffe60 0x7fd90819a8a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:15.510 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90e76d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd908100810 0x7fd90819cef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:15.510 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90e76d700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd90819d6c0 con 0x7fd908100810 2023-03-27T10:22:15.510 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90e76d700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd90819d830 con 0x7fd9080febd0 2023-03-27T10:22:15.511 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90e76d700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd90819d9f0 con 0x7fd9080ffe60 2023-03-27T10:22:15.511 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90cd0a700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd908100810 0x7fd90819cef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:15.511 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90cd0a700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd908100810 0x7fd90819cef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45452/0 (socket says 172.21.15.159:45452) 2023-03-27T10:22:15.511 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90cd0a700 1 -- 172.21.15.159:0/2690712023 learned_addr learned my addr 172.21.15.159:0/2690712023 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:15.511 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd907fff700 1 --2- 172.21.15.159:0/2690712023 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd9080febd0 0x7fd908068450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:15.511 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd9077fe700 1 --2- 172.21.15.159:0/2690712023 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd9080ffe60 0x7fd90819a8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:15.511 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90cd0a700 1 -- 172.21.15.159:0/2690712023 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd9080ffe60 msgr2=0x7fd90819a8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:15.511 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90cd0a700 1 --2- 172.21.15.159:0/2690712023 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd9080ffe60 0x7fd90819a8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:15.512 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90cd0a700 1 -- 172.21.15.159:0/2690712023 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd9080febd0 msgr2=0x7fd908068450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:15.512 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90cd0a700 1 --2- 172.21.15.159:0/2690712023 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd9080febd0 0x7fd908068450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:15.512 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90cd0a700 1 -- 172.21.15.159:0/2690712023 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd900009d00 con 0x7fd908100810 2023-03-27T10:22:15.512 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd9077fe700 1 --2- 172.21.15.159:0/2690712023 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd9080ffe60 0x7fd90819a8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:22:15.512 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.508+0000 7fd90cd0a700 1 --2- 172.21.15.159:0/2690712023 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd908100810 0x7fd90819cef0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7fd90000b780 tx=0x7fd90000bd60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:15.512 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.509+0000 7fd9057fa700 1 -- 172.21.15.159:0/2690712023 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd90000e040 con 0x7fd908100810 2023-03-27T10:22:15.512 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.509+0000 7fd9057fa700 1 -- 172.21.15.159:0/2690712023 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd9000094a0 con 0x7fd908100810 2023-03-27T10:22:15.512 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.509+0000 7fd90e76d700 1 -- 172.21.15.159:0/2690712023 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd90810a530 con 0x7fd908100810 2023-03-27T10:22:15.513 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.509+0000 7fd9057fa700 1 -- 172.21.15.159:0/2690712023 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd900005330 con 0x7fd908100810 2023-03-27T10:22:15.513 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.509+0000 7fd90e76d700 1 -- 172.21.15.159:0/2690712023 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd90810a3a0 con 0x7fd908100810 2023-03-27T10:22:15.513 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.510+0000 7fd90e76d700 1 -- 172.21.15.159:0/2690712023 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd908105ba0 con 0x7fd908100810 2023-03-27T10:22:15.513 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.511+0000 7fd9057fa700 1 -- 172.21.15.159:0/2690712023 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7fd90001da30 con 0x7fd908100810 2023-03-27T10:22:15.514 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.511+0000 7fd9057fa700 1 --2- 172.21.15.159:0/2690712023 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd8f009b420 0x7fd8f009d8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:15.514 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.511+0000 7fd907fff700 1 --2- 172.21.15.159:0/2690712023 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd8f009b420 0x7fd8f009d8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:15.514 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.512+0000 7fd9057fa700 1 -- 172.21.15.159:0/2690712023 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7fd9000c61b0 con 0x7fd908100810 2023-03-27T10:22:15.514 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.512+0000 7fd907fff700 1 --2- 172.21.15.159:0/2690712023 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd8f009b420 0x7fd8f009d8d0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7fd8f8009c60 tx=0x7fd8f8009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:15.515 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.516+0000 7fd9057fa700 1 -- 172.21.15.159:0/2690712023 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fd900092050 con 0x7fd908100810 2023-03-27T10:22:15.739 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.740+0000 7fd90e76d700 1 -- 172.21.15.159:0/2690712023 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd90802cdb0 con 0x7fd908100810 2023-03-27T10:22:15.740 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.740+0000 7fd9057fa700 1 -- 172.21.15.159:0/2690712023 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 8 v8) v1 ==== 93+0+5817 (secure 0 0 0) 0x7fd900095670 con 0x7fd908100810 2023-03-27T10:22:15.740 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:15.740 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":8,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14556,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6835/3048624466","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6834","nonce":3048624466},{"type":"v1","addr":"172.21.15.159:6835","nonce":3048624466}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14562,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6837/1998956589","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6836","nonce":1998956589},{"type":"v1","addr":"172.21.15.159:6837","nonce":1998956589}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24424,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6833/1022778932","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6832","nonce":1022778932},{"type":"v1","addr":"172.21.15.171:6833","nonce":1022778932}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24440,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6833/2969583384","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6832","nonce":2969583384},{"type":"v1","addr":"172.21.15.161:6833","nonce":2969583384}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24464,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6835/761787339","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6834","nonce":761787339},{"type":"v1","addr":"172.21.15.161:6835","nonce":761787339}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24472,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6835/1823903399","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6834","nonce":1823903399},{"type":"v1","addr":"172.21.15.171:6835","nonce":1823903399}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24484,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6837/2737069329","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6836","nonce":2737069329},{"type":"v1","addr":"172.21.15.171:6837","nonce":2737069329}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8}],"filesystems":[]} 2023-03-27T10:22:15.742 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.743+0000 7fd90e76d700 1 -- 172.21.15.159:0/2690712023 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd8f009b420 msgr2=0x7fd8f009d8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:15.742 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.743+0000 7fd90e76d700 1 --2- 172.21.15.159:0/2690712023 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd8f009b420 0x7fd8f009d8d0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7fd8f8009c60 tx=0x7fd8f8009260 comp rx=0 tx=0).stop 2023-03-27T10:22:15.742 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.743+0000 7fd90e76d700 1 -- 172.21.15.159:0/2690712023 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd908100810 msgr2=0x7fd90819cef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:15.742 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.743+0000 7fd90e76d700 1 --2- 172.21.15.159:0/2690712023 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd908100810 0x7fd90819cef0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7fd90000b780 tx=0x7fd90000bd60 comp rx=0 tx=0).stop 2023-03-27T10:22:15.743 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.743+0000 7fd90e76d700 1 -- 172.21.15.159:0/2690712023 shutdown_connections 2023-03-27T10:22:15.743 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.743+0000 7fd90e76d700 1 --2- 172.21.15.159:0/2690712023 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fd8f009b420 0x7fd8f009d8d0 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:15.743 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.743+0000 7fd90e76d700 1 --2- 172.21.15.159:0/2690712023 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fd908100810 0x7fd90819cef0 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:15.743 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.743+0000 7fd90e76d700 1 --2- 172.21.15.159:0/2690712023 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fd9080febd0 0x7fd908068450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:15.743 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.743+0000 7fd90e76d700 1 --2- 172.21.15.159:0/2690712023 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fd9080ffe60 0x7fd90819a8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:15.743 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.743+0000 7fd90e76d700 1 -- 172.21.15.159:0/2690712023 >> 172.21.15.159:0/2690712023 conn(0x7fd9080fa210 msgr2=0x7fd9080fc4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:15.743 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.743+0000 7fd90e76d700 1 -- 172.21.15.159:0/2690712023 shutdown_connections 2023-03-27T10:22:15.743 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:15.743+0000 7fd90e76d700 1 -- 172.21.15.159:0/2690712023 wait complete. 2023-03-27T10:22:15.744 INFO:teuthology.orchestra.run.smithi159.stderr:dumped fsmap epoch 8 2023-03-27T10:22:16.047 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:15 smithi161 ceph-mon[94333]: mds.? [v2:172.21.15.171:6836/2737069329,v1:172.21.15.171:6837/2737069329] up:boot 2023-03-27T10:22:16.047 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:15 smithi161 ceph-mon[94333]: fsmap 7 up:standby 2023-03-27T10:22:16.047 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:15 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2023-03-27T10:22:16.047 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:15 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:16.048 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:15 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:16.048 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:15 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:16.048 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:15 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:16.048 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:15 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:16.048 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:15 smithi161 ceph-mon[94333]: Deploying daemon mds.h on smithi161 2023-03-27T10:22:16.048 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:15 smithi161 ceph-mon[94333]: pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:16.048 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:15 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/2690712023' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:16.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:15 smithi159 ceph-mon[93039]: mds.? [v2:172.21.15.171:6836/2737069329,v1:172.21.15.171:6837/2737069329] up:boot 2023-03-27T10:22:16.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:15 smithi159 ceph-mon[93039]: fsmap 7 up:standby 2023-03-27T10:22:16.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:15 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2023-03-27T10:22:16.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:15 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:16.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:15 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:16.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:15 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:16.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:15 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:16.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:15 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:16.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:15 smithi159 ceph-mon[93039]: Deploying daemon mds.h on smithi161 2023-03-27T10:22:16.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:15 smithi159 ceph-mon[93039]: pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:16.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:15 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/2690712023' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:16.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:15 smithi171 ceph-mon[94808]: mds.? [v2:172.21.15.171:6836/2737069329,v1:172.21.15.171:6837/2737069329] up:boot 2023-03-27T10:22:16.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:15 smithi171 ceph-mon[94808]: fsmap 7 up:standby 2023-03-27T10:22:16.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:15 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2023-03-27T10:22:16.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:15 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:16.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:15 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:16.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:15 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:16.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:15 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:16.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:15 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:16.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:15 smithi171 ceph-mon[94808]: Deploying daemon mds.h on smithi161 2023-03-27T10:22:16.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:15 smithi171 ceph-mon[94808]: pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:16.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:15 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/2690712023' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:16.584 INFO:journalctl@ceph.mds.h.smithi161.stdout:Mar 27 10:22:16 smithi161 systemd[1]: Starting Ceph mds.h for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:16.878 INFO:journalctl@ceph.mds.h.smithi161.stdout:Mar 27 10:22:16 smithi161 podman[119074]: 2023-03-27T10:22:17.334 INFO:journalctl@ceph.mds.h.smithi161.stdout:Mar 27 10:22:16 smithi161 bash[119074]: a9f0a5b96ff96a40487f44018dc62c1df0d10063d70ee5ccc156dabe8d475b34 2023-03-27T10:22:17.334 INFO:journalctl@ceph.mds.h.smithi161.stdout:Mar 27 10:22:16 smithi161 ceph-mds[119114]: -- 172.21.15.161:0/3225368824 <== mon.2 v2:172.21.15.161:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558eefeaeb60 con 0x558eefe84c00 2023-03-27T10:22:17.334 INFO:journalctl@ceph.mds.h.smithi161.stdout:Mar 27 10:22:16 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-h[119090]: starting mds.h at 2023-03-27T10:22:17.334 INFO:journalctl@ceph.mds.h.smithi161.stdout:Mar 27 10:22:16 smithi161 systemd[1]: Started Ceph mds.h for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:22:17.755 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-03-27T10:22:17.890 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.890+0000 7f7d863a2700 1 -- 172.21.15.159:0/4027731801 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d800fea40 msgr2=0x7f7d800fee50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:17.890 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.890+0000 7f7d863a2700 1 --2- 172.21.15.159:0/4027731801 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d800fea40 0x7f7d800fee50 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f7d70009990 tx=0x7f7d70009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:17.890 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.890+0000 7f7d863a2700 1 -- 172.21.15.159:0/4027731801 shutdown_connections 2023-03-27T10:22:17.890 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.890+0000 7f7d863a2700 1 --2- 172.21.15.159:0/4027731801 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7d801005c0 0x7f7d80106e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:17.890 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.890+0000 7f7d863a2700 1 --2- 172.21.15.159:0/4027731801 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d800fea40 0x7f7d800fee50 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:17.890 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.890+0000 7f7d863a2700 1 --2- 172.21.15.159:0/4027731801 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7d800ffc30 0x7f7d80100080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:17.891 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.890+0000 7f7d863a2700 1 -- 172.21.15.159:0/4027731801 >> 172.21.15.159:0/4027731801 conn(0x7f7d800fa060 msgr2=0x7f7d800fc490 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:17.891 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.890+0000 7f7d863a2700 1 -- 172.21.15.159:0/4027731801 shutdown_connections 2023-03-27T10:22:17.891 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.890+0000 7f7d863a2700 1 -- 172.21.15.159:0/4027731801 wait complete. 2023-03-27T10:22:17.891 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d863a2700 1 Processor -- start 2023-03-27T10:22:17.893 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d863a2700 1 -- start start 2023-03-27T10:22:17.893 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d863a2700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7d800fea40 0x7f7d80068430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:17.893 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d863a2700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d800ffc30 0x7f7d8019a5d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:17.893 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d863a2700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7d801005c0 0x7f7d8019cc10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:17.893 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d863a2700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d8019d3b0 con 0x7f7d800ffc30 2023-03-27T10:22:17.893 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d863a2700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d8019d520 con 0x7f7d801005c0 2023-03-27T10:22:17.894 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d863a2700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d8019d6e0 con 0x7f7d800fea40 2023-03-27T10:22:17.894 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d84b9f700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d800ffc30 0x7f7d8019a5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:17.894 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d853a0700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7d800fea40 0x7f7d80068430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:17.894 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d84b9f700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d800ffc30 0x7f7d8019a5d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45464/0 (socket says 172.21.15.159:45464) 2023-03-27T10:22:17.894 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d84b9f700 1 -- 172.21.15.159:0/4172497359 learned_addr learned my addr 172.21.15.159:0/4172497359 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:17.894 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d853a0700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7d800fea40 0x7f7d80068430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.171:3300/0 says I am v2:172.21.15.159:46000/0 (socket says 172.21.15.159:46000) 2023-03-27T10:22:17.894 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d85ba1700 1 --2- 172.21.15.159:0/4172497359 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7d801005c0 0x7f7d8019cc10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:17.895 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d84b9f700 1 -- 172.21.15.159:0/4172497359 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7d800fea40 msgr2=0x7f7d80068430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:17.895 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d84b9f700 1 --2- 172.21.15.159:0/4172497359 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7d800fea40 0x7f7d80068430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:17.895 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d84b9f700 1 -- 172.21.15.159:0/4172497359 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7d801005c0 msgr2=0x7f7d8019cc10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:17.895 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d84b9f700 1 --2- 172.21.15.159:0/4172497359 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7d801005c0 0x7f7d8019cc10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:17.895 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d84b9f700 1 -- 172.21.15.159:0/4172497359 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d700095f0 con 0x7f7d800ffc30 2023-03-27T10:22:17.895 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d85ba1700 1 --2- 172.21.15.159:0/4172497359 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7d801005c0 0x7f7d8019cc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:22:17.895 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d84b9f700 1 --2- 172.21.15.159:0/4172497359 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d800ffc30 0x7f7d8019a5d0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f7d7c00d980 tx=0x7f7d7c00dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:17.895 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.891+0000 7f7d767fc700 1 -- 172.21.15.159:0/4172497359 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d7c004220 con 0x7f7d800ffc30 2023-03-27T10:22:17.896 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.892+0000 7f7d767fc700 1 -- 172.21.15.159:0/4172497359 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7d7c007640 con 0x7f7d800ffc30 2023-03-27T10:22:17.896 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.892+0000 7f7d767fc700 1 -- 172.21.15.159:0/4172497359 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d7c005610 con 0x7f7d800ffc30 2023-03-27T10:22:17.896 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.892+0000 7f7d863a2700 1 -- 172.21.15.159:0/4172497359 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7d8010a3a0 con 0x7f7d800ffc30 2023-03-27T10:22:17.896 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.892+0000 7f7d863a2700 1 -- 172.21.15.159:0/4172497359 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7d8010a210 con 0x7f7d800ffc30 2023-03-27T10:22:17.896 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.893+0000 7f7d863a2700 1 -- 172.21.15.159:0/4172497359 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7d80105970 con 0x7f7d800ffc30 2023-03-27T10:22:17.896 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.894+0000 7f7d767fc700 1 -- 172.21.15.159:0/4172497359 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f7d7c021070 con 0x7f7d800ffc30 2023-03-27T10:22:17.897 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.894+0000 7f7d767fc700 1 --2- 172.21.15.159:0/4172497359 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7d6c09b420 0x7f7d6c09d8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:17.897 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.894+0000 7f7d853a0700 1 --2- 172.21.15.159:0/4172497359 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7d6c09b420 0x7f7d6c09d8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:17.897 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.894+0000 7f7d767fc700 1 -- 172.21.15.159:0/4172497359 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f7d7c0c4c50 con 0x7f7d800ffc30 2023-03-27T10:22:17.897 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.895+0000 7f7d853a0700 1 --2- 172.21.15.159:0/4172497359 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7d6c09b420 0x7f7d6c09d8d0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f7d7000b5c0 tx=0x7f7d70005960 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:17.898 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:17.897+0000 7f7d767fc700 1 -- 172.21.15.159:0/4172497359 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f7d7c090a60 con 0x7f7d800ffc30 2023-03-27T10:22:18.022 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:17 smithi171 ceph-mon[94808]: mds.? [v2:172.21.15.161:6836/406989937,v1:172.21.15.161:6837/406989937] up:boot 2023-03-27T10:22:18.023 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:17 smithi171 ceph-mon[94808]: fsmap 8 up:standby 2023-03-27T10:22:18.023 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:17 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2023-03-27T10:22:18.023 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:17 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:18.023 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:17 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:18.023 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:17 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:18.023 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:17 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:18.023 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:17 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:18.024 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:17 smithi171 ceph-mon[94808]: Deploying daemon mds.i on smithi171 2023-03-27T10:22:18.024 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:17 smithi171 ceph-mon[94808]: pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:18.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:17 smithi159 ceph-mon[93039]: mds.? [v2:172.21.15.161:6836/406989937,v1:172.21.15.161:6837/406989937] up:boot 2023-03-27T10:22:18.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:17 smithi159 ceph-mon[93039]: fsmap 8 up:standby 2023-03-27T10:22:18.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:17 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2023-03-27T10:22:18.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:17 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:18.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:17 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:18.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:17 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:18.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:17 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:18.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:17 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:18.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:17 smithi159 ceph-mon[93039]: Deploying daemon mds.i on smithi171 2023-03-27T10:22:18.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:17 smithi159 ceph-mon[93039]: pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:18.139 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:18.140+0000 7f7d863a2700 1 -- 172.21.15.159:0/4172497359 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f7d8002cdb0 con 0x7f7d800ffc30 2023-03-27T10:22:18.140 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:18.140+0000 7f7d767fc700 1 -- 172.21.15.159:0/4172497359 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 9 v9) v1 ==== 93+0+6574 (secure 0 0 0) 0x7f7d7c016020 con 0x7f7d800ffc30 2023-03-27T10:22:18.140 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:18.140 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":9,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14556,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6835/3048624466","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6834","nonce":3048624466},{"type":"v1","addr":"172.21.15.159:6835","nonce":3048624466}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14562,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6837/1998956589","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6836","nonce":1998956589},{"type":"v1","addr":"172.21.15.159:6837","nonce":1998956589}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24424,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6833/1022778932","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6832","nonce":1022778932},{"type":"v1","addr":"172.21.15.171:6833","nonce":1022778932}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24440,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6833/2969583384","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6832","nonce":2969583384},{"type":"v1","addr":"172.21.15.161:6833","nonce":2969583384}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24464,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6835/761787339","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6834","nonce":761787339},{"type":"v1","addr":"172.21.15.161:6835","nonce":761787339}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24472,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6835/1823903399","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6834","nonce":1823903399},{"type":"v1","addr":"172.21.15.171:6835","nonce":1823903399}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24484,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6837/2737069329","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6836","nonce":2737069329},{"type":"v1","addr":"172.21.15.171:6837","nonce":2737069329}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24491,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6837/406989937","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6836","nonce":406989937},{"type":"v1","addr":"172.21.15.161:6837","nonce":406989937}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9}],"filesystems":[]} 2023-03-27T10:22:18.142 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:18.142+0000 7f7d863a2700 1 -- 172.21.15.159:0/4172497359 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7d6c09b420 msgr2=0x7f7d6c09d8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:18.142 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:18.142+0000 7f7d863a2700 1 --2- 172.21.15.159:0/4172497359 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7d6c09b420 0x7f7d6c09d8d0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f7d7000b5c0 tx=0x7f7d70005960 comp rx=0 tx=0).stop 2023-03-27T10:22:18.143 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:18.142+0000 7f7d863a2700 1 -- 172.21.15.159:0/4172497359 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d800ffc30 msgr2=0x7f7d8019a5d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:18.143 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:18.142+0000 7f7d863a2700 1 --2- 172.21.15.159:0/4172497359 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d800ffc30 0x7f7d8019a5d0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f7d7c00d980 tx=0x7f7d7c00dc90 comp rx=0 tx=0).stop 2023-03-27T10:22:18.143 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:18.142+0000 7f7d863a2700 1 -- 172.21.15.159:0/4172497359 shutdown_connections 2023-03-27T10:22:18.143 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:18.142+0000 7f7d863a2700 1 --2- 172.21.15.159:0/4172497359 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f7d6c09b420 0x7f7d6c09d8d0 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:18.143 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:18.142+0000 7f7d863a2700 1 --2- 172.21.15.159:0/4172497359 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7d800fea40 0x7f7d80068430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:18.144 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:18.142+0000 7f7d863a2700 1 --2- 172.21.15.159:0/4172497359 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7d801005c0 0x7f7d8019cc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:18.144 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:18.143+0000 7f7d863a2700 1 --2- 172.21.15.159:0/4172497359 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7d800ffc30 0x7f7d8019a5d0 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:18.144 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:18.143+0000 7f7d863a2700 1 -- 172.21.15.159:0/4172497359 >> 172.21.15.159:0/4172497359 conn(0x7f7d800fa060 msgr2=0x7f7d800fc280 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:18.144 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:18.143+0000 7f7d863a2700 1 -- 172.21.15.159:0/4172497359 shutdown_connections 2023-03-27T10:22:18.144 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:18.143+0000 7f7d863a2700 1 -- 172.21.15.159:0/4172497359 wait complete. 2023-03-27T10:22:18.144 INFO:teuthology.orchestra.run.smithi159.stderr:dumped fsmap epoch 9 2023-03-27T10:22:18.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:17 smithi161 ceph-mon[94333]: mds.? [v2:172.21.15.161:6836/406989937,v1:172.21.15.161:6837/406989937] up:boot 2023-03-27T10:22:18.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:17 smithi161 ceph-mon[94333]: fsmap 8 up:standby 2023-03-27T10:22:18.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:17 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2023-03-27T10:22:18.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:17 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:18.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:17 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:18.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:17 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:18.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:17 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:18.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:17 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:18.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:17 smithi161 ceph-mon[94333]: Deploying daemon mds.i on smithi171 2023-03-27T10:22:18.335 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:17 smithi161 ceph-mon[94333]: pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:18.726 INFO:journalctl@ceph.mds.i.smithi171.stdout:Mar 27 10:22:18 smithi171 systemd[1]: Starting Ceph mds.i for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:19.020 INFO:journalctl@ceph.mds.i.smithi171.stdout:Mar 27 10:22:18 smithi171 podman[120897]: 2023-03-27T10:22:19.020 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:18 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/4172497359' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:19.334 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:18 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/4172497359' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:19.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:18 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/4172497359' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:19.476 INFO:journalctl@ceph.mds.i.smithi171.stdout:Mar 27 10:22:19 smithi171 ceph-mds[120937]: -- 172.21.15.171:0/3820095978 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f66b0f2b60 con 0x55f66b0c9000 2023-03-27T10:22:19.476 INFO:journalctl@ceph.mds.i.smithi171.stdout:Mar 27 10:22:19 smithi171 bash[120897]: b8a5b35ea0c28d26972a29b9d7a9b3d5bebffd90214dcbd2c9f6ceef040ced1c 2023-03-27T10:22:19.476 INFO:journalctl@ceph.mds.i.smithi171.stdout:Mar 27 10:22:19 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-i[120913]: starting mds.i at 2023-03-27T10:22:19.476 INFO:journalctl@ceph.mds.i.smithi171.stdout:Mar 27 10:22:19 smithi171 systemd[1]: Started Ceph mds.i for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:22:20.155 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-03-27T10:22:20.167 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:20 smithi159 ceph-mon[93039]: pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:20.168 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:20 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:20.168 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:20 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:20.168 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:20 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:20.168 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:20 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:20.168 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:20 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:20.168 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:20 smithi159 ceph-mon[93039]: Deploying daemon mds.g on smithi159 2023-03-27T10:22:20.169 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:20 smithi159 ceph-mon[93039]: mds.? [v2:172.21.15.171:6838/1538859602,v1:172.21.15.171:6839/1538859602] up:boot 2023-03-27T10:22:20.169 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:20 smithi159 ceph-mon[93039]: fsmap 9 up:standby 2023-03-27T10:22:20.169 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:20 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2023-03-27T10:22:20.281 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.282+0000 7f8dc13c7700 1 -- 172.21.15.159:0/977650842 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8dbc0febf0 msgr2=0x7f8dbc0ff000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:20.281 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.282+0000 7f8dc13c7700 1 --2- 172.21.15.159:0/977650842 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8dbc0febf0 0x7f8dbc0ff000 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f8dac009990 tx=0x7f8dac009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:20.282 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.282+0000 7f8dc13c7700 1 -- 172.21.15.159:0/977650842 shutdown_connections 2023-03-27T10:22:20.282 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.282+0000 7f8dc13c7700 1 --2- 172.21.15.159:0/977650842 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f8dbc100770 0x7f8dbc107030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:20.282 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.282+0000 7f8dc13c7700 1 --2- 172.21.15.159:0/977650842 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8dbc0febf0 0x7f8dbc0ff000 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:20.282 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.282+0000 7f8dc13c7700 1 --2- 172.21.15.159:0/977650842 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f8dbc0ffde0 0x7f8dbc100230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:20.282 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.282+0000 7f8dc13c7700 1 -- 172.21.15.159:0/977650842 >> 172.21.15.159:0/977650842 conn(0x7f8dbc0fa1f0 msgr2=0x7f8dbc0fc640 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:20.282 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.282+0000 7f8dc13c7700 1 -- 172.21.15.159:0/977650842 shutdown_connections 2023-03-27T10:22:20.282 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.282+0000 7f8dc13c7700 1 -- 172.21.15.159:0/977650842 wait complete. 2023-03-27T10:22:20.284 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.282+0000 7f8dc13c7700 1 Processor -- start 2023-03-27T10:22:20.284 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.282+0000 7f8dc13c7700 1 -- start start 2023-03-27T10:22:20.284 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dc13c7700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f8dbc0febf0 0x7f8dbc194970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:20.284 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dc13c7700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8dbc0ffde0 0x7f8dbc194eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:20.285 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dc13c7700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f8dbc100770 0x7f8dbc19bf20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:20.285 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dc13c7700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8dbc19c4f0 con 0x7f8dbc0ffde0 2023-03-27T10:22:20.285 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dc13c7700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8dbc19c630 con 0x7f8dbc100770 2023-03-27T10:22:20.285 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dc13c7700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8dbc19c7c0 con 0x7f8dbc0febf0 2023-03-27T10:22:20.285 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dba7fc700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8dbc0ffde0 0x7f8dbc194eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:20.285 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dba7fc700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8dbc0ffde0 0x7f8dbc194eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45478/0 (socket says 172.21.15.159:45478) 2023-03-27T10:22:20.285 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dba7fc700 1 -- 172.21.15.159:0/3349151039 learned_addr learned my addr 172.21.15.159:0/3349151039 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:20.286 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dbaffd700 1 --2- 172.21.15.159:0/3349151039 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f8dbc0febf0 0x7f8dbc194970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:20.286 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dbb7fe700 1 --2- 172.21.15.159:0/3349151039 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f8dbc100770 0x7f8dbc19bf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:20.286 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dba7fc700 1 -- 172.21.15.159:0/3349151039 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f8dbc0febf0 msgr2=0x7f8dbc194970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:20.286 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dba7fc700 1 --2- 172.21.15.159:0/3349151039 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f8dbc0febf0 0x7f8dbc194970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:20.286 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dba7fc700 1 -- 172.21.15.159:0/3349151039 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f8dbc100770 msgr2=0x7f8dbc19bf20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:20.286 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dba7fc700 1 --2- 172.21.15.159:0/3349151039 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f8dbc100770 0x7f8dbc19bf20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:20.286 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dba7fc700 1 -- 172.21.15.159:0/3349151039 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8dac0095f0 con 0x7f8dbc0ffde0 2023-03-27T10:22:20.287 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dba7fc700 1 --2- 172.21.15.159:0/3349151039 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8dbc0ffde0 0x7f8dbc194eb0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f8db000d980 tx=0x7f8db000dd40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:20.287 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8da3fff700 1 -- 172.21.15.159:0/3349151039 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8db0009b40 con 0x7f8dbc0ffde0 2023-03-27T10:22:20.287 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dc13c7700 1 -- 172.21.15.159:0/3349151039 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8dbc10a570 con 0x7f8dbc0ffde0 2023-03-27T10:22:20.287 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8da3fff700 1 -- 172.21.15.159:0/3349151039 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8db000de60 con 0x7f8dbc0ffde0 2023-03-27T10:22:20.287 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8da3fff700 1 -- 172.21.15.159:0/3349151039 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8db0005580 con 0x7f8dbc0ffde0 2023-03-27T10:22:20.287 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.283+0000 7f8dc13c7700 1 -- 172.21.15.159:0/3349151039 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8dbc10a3e0 con 0x7f8dbc0ffde0 2023-03-27T10:22:20.287 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.284+0000 7f8dc13c7700 1 -- 172.21.15.159:0/3349151039 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8dbc105b40 con 0x7f8dbc0ffde0 2023-03-27T10:22:20.288 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.285+0000 7f8da3fff700 1 -- 172.21.15.159:0/3349151039 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f8db0022070 con 0x7f8dbc0ffde0 2023-03-27T10:22:20.288 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.286+0000 7f8da3fff700 1 --2- 172.21.15.159:0/3349151039 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f8da409b3a0 0x7f8da409d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:20.288 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.286+0000 7f8da3fff700 1 -- 172.21.15.159:0/3349151039 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f8db00c48d0 con 0x7f8dbc0ffde0 2023-03-27T10:22:20.288 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.286+0000 7f8dbaffd700 1 --2- 172.21.15.159:0/3349151039 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f8da409b3a0 0x7f8da409d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:20.289 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.286+0000 7f8dbaffd700 1 --2- 172.21.15.159:0/3349151039 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f8da409b3a0 0x7f8da409d850 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f8dac006010 tx=0x7f8dac00b540 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:20.289 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.290+0000 7f8da3fff700 1 -- 172.21.15.159:0/3349151039 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f8db00906e0 con 0x7f8dbc0ffde0 2023-03-27T10:22:20.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:20 smithi171 ceph-mon[94808]: pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:20.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:20 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:20.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:20 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:20.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:20 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:20.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:20 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:20.476 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:20 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:20.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:20 smithi171 ceph-mon[94808]: Deploying daemon mds.g on smithi159 2023-03-27T10:22:20.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:20 smithi171 ceph-mon[94808]: mds.? [v2:172.21.15.171:6838/1538859602,v1:172.21.15.171:6839/1538859602] up:boot 2023-03-27T10:22:20.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:20 smithi171 ceph-mon[94808]: fsmap 9 up:standby 2023-03-27T10:22:20.477 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:20 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2023-03-27T10:22:20.560 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.560+0000 7f8dc13c7700 1 -- 172.21.15.159:0/3349151039 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8dbc061810 con 0x7f8dbc0ffde0 2023-03-27T10:22:20.561 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.561+0000 7f8da3fff700 1 -- 172.21.15.159:0/3349151039 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 10 v10) v1 ==== 94+0+7336 (secure 0 0 0) 0x7f8db00a9cc0 con 0x7f8dbc0ffde0 2023-03-27T10:22:20.561 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:20.561 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":10,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14556,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6835/3048624466","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6834","nonce":3048624466},{"type":"v1","addr":"172.21.15.159:6835","nonce":3048624466}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14562,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6837/1998956589","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6836","nonce":1998956589},{"type":"v1","addr":"172.21.15.159:6837","nonce":1998956589}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24424,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6833/1022778932","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6832","nonce":1022778932},{"type":"v1","addr":"172.21.15.171:6833","nonce":1022778932}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24440,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6833/2969583384","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6832","nonce":2969583384},{"type":"v1","addr":"172.21.15.161:6833","nonce":2969583384}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24464,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6835/761787339","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6834","nonce":761787339},{"type":"v1","addr":"172.21.15.161:6835","nonce":761787339}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24472,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6835/1823903399","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6834","nonce":1823903399},{"type":"v1","addr":"172.21.15.171:6835","nonce":1823903399}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24484,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6837/2737069329","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6836","nonce":2737069329},{"type":"v1","addr":"172.21.15.171:6837","nonce":2737069329}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24491,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6837/406989937","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6836","nonce":406989937},{"type":"v1","addr":"172.21.15.161:6837","nonce":406989937}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9},{"gid":24502,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6839/1538859602","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6838","nonce":1538859602},{"type":"v1","addr":"172.21.15.171:6839","nonce":1538859602}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":10}],"filesystems":[]} 2023-03-27T10:22:20.563 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.564+0000 7f8dc13c7700 1 -- 172.21.15.159:0/3349151039 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f8da409b3a0 msgr2=0x7f8da409d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:20.563 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.564+0000 7f8dc13c7700 1 --2- 172.21.15.159:0/3349151039 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f8da409b3a0 0x7f8da409d850 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f8dac006010 tx=0x7f8dac00b540 comp rx=0 tx=0).stop 2023-03-27T10:22:20.563 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.564+0000 7f8dc13c7700 1 -- 172.21.15.159:0/3349151039 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8dbc0ffde0 msgr2=0x7f8dbc194eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:20.563 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.564+0000 7f8dc13c7700 1 --2- 172.21.15.159:0/3349151039 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8dbc0ffde0 0x7f8dbc194eb0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f8db000d980 tx=0x7f8db000dd40 comp rx=0 tx=0).stop 2023-03-27T10:22:20.564 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.564+0000 7f8dc13c7700 1 -- 172.21.15.159:0/3349151039 shutdown_connections 2023-03-27T10:22:20.564 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.564+0000 7f8dc13c7700 1 --2- 172.21.15.159:0/3349151039 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f8da409b3a0 0x7f8da409d850 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:20.564 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.564+0000 7f8dc13c7700 1 --2- 172.21.15.159:0/3349151039 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f8dbc0febf0 0x7f8dbc194970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:20.564 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.564+0000 7f8dc13c7700 1 --2- 172.21.15.159:0/3349151039 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f8dbc100770 0x7f8dbc19bf20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:20.564 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.564+0000 7f8dc13c7700 1 --2- 172.21.15.159:0/3349151039 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f8dbc0ffde0 0x7f8dbc194eb0 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:20.565 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.564+0000 7f8dc13c7700 1 -- 172.21.15.159:0/3349151039 >> 172.21.15.159:0/3349151039 conn(0x7f8dbc0fa1f0 msgr2=0x7f8dbc0fc490 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:20.565 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.564+0000 7f8dc13c7700 1 -- 172.21.15.159:0/3349151039 shutdown_connections 2023-03-27T10:22:20.565 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:20.564+0000 7f8dc13c7700 1 -- 172.21.15.159:0/3349151039 wait complete. 2023-03-27T10:22:20.565 INFO:teuthology.orchestra.run.smithi159.stderr:dumped fsmap epoch 10 2023-03-27T10:22:20.583 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:20 smithi161 ceph-mon[94333]: pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:20.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:20 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:20.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:20 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:20.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:20 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:20.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:20 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:20.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:20 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:20.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:20 smithi161 ceph-mon[94333]: Deploying daemon mds.g on smithi159 2023-03-27T10:22:20.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:20 smithi161 ceph-mon[94333]: mds.? [v2:172.21.15.171:6838/1538859602,v1:172.21.15.171:6839/1538859602] up:boot 2023-03-27T10:22:20.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:20 smithi161 ceph-mon[94333]: fsmap 9 up:standby 2023-03-27T10:22:20.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:20 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2023-03-27T10:22:20.865 INFO:journalctl@ceph.mds.g.smithi159.stdout:Mar 27 10:22:20 smithi159 systemd[1]: Starting Ceph mds.g for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:21.231 INFO:journalctl@ceph.mds.g.smithi159.stdout:Mar 27 10:22:20 smithi159 podman[127246]: 2023-03-27T10:22:21.231 INFO:journalctl@ceph.mds.g.smithi159.stdout:Mar 27 10:22:21 smithi159 ceph-mds[127286]: -- 172.21.15.159:0/3374404895 <== mon.0 v2:172.21.15.159:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b792886b60 con 0x55b79285d000 2023-03-27T10:22:21.231 INFO:journalctl@ceph.mds.g.smithi159.stdout:Mar 27 10:22:21 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-g[127262]: starting mds.g at 2023-03-27T10:22:21.231 INFO:journalctl@ceph.mds.g.smithi159.stdout:Mar 27 10:22:21 smithi159 bash[127246]: a7ceb296cdd4677d27586a3578eaa5d1b7653c92c7e028a37810757e17826134 2023-03-27T10:22:21.615 INFO:journalctl@ceph.mds.g.smithi159.stdout:Mar 27 10:22:21 smithi159 systemd[1]: Started Ceph mds.g for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:22:21.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:21 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3349151039' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:21.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:21 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:21.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:21 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:21.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:21 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:21.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:21 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:21.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:21 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:21.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:21 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3349151039' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:21.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:21 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:21.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:21 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:21.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:21 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:21.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:21 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:21.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:21 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:21.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:21 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3349151039' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:21.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:21 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:21.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:21 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:21.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:21 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:21.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:21 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:21.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:21 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:22.577 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-03-27T10:22:22.604 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:22 smithi159 ceph-mon[93039]: pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:22.605 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:22 smithi159 ceph-mon[93039]: Deploying daemon mds.a on smithi159 2023-03-27T10:22:22.605 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:22 smithi159 ceph-mon[93039]: mds.? [v2:172.21.15.159:6838/793152328,v1:172.21.15.159:6839/793152328] up:boot 2023-03-27T10:22:22.605 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:22 smithi159 ceph-mon[93039]: fsmap 10 up:standby 2023-03-27T10:22:22.605 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:22 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2023-03-27T10:22:22.718 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.719+0000 7fef75865700 1 -- 172.21.15.159:0/1424470354 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fef70101870 msgr2=0x7fef70108130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:22.718 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.719+0000 7fef75865700 1 --2- 172.21.15.159:0/1424470354 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fef70101870 0x7fef70108130 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7fef68009f90 tx=0x7fef6800b330 comp rx=0 tx=0).stop 2023-03-27T10:22:22.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.719+0000 7fef75865700 1 -- 172.21.15.159:0/1424470354 shutdown_connections 2023-03-27T10:22:22.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.719+0000 7fef75865700 1 --2- 172.21.15.159:0/1424470354 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fef70101870 0x7fef70108130 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:22.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.719+0000 7fef75865700 1 --2- 172.21.15.159:0/1424470354 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fef700ffcf0 0x7fef70100100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:22.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.719+0000 7fef75865700 1 --2- 172.21.15.159:0/1424470354 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fef70100ee0 0x7fef70101330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:22.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.719+0000 7fef75865700 1 -- 172.21.15.159:0/1424470354 >> 172.21.15.159:0/1424470354 conn(0x7fef700fb2f0 msgr2=0x7fef700fd740 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:22.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.719+0000 7fef75865700 1 -- 172.21.15.159:0/1424470354 shutdown_connections 2023-03-27T10:22:22.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.719+0000 7fef75865700 1 -- 172.21.15.159:0/1424470354 wait complete. 2023-03-27T10:22:22.721 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.720+0000 7fef75865700 1 Processor -- start 2023-03-27T10:22:22.721 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.720+0000 7fef75865700 1 -- start start 2023-03-27T10:22:22.721 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.720+0000 7fef75865700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fef700ffcf0 0x7fef70195b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:22.722 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.720+0000 7fef75865700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fef70100ee0 0x7fef70196060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:22.722 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.720+0000 7fef75865700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fef70101870 0x7fef7019d0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:22.722 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.720+0000 7fef75865700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fef7019d6a0 con 0x7fef70100ee0 2023-03-27T10:22:22.722 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.720+0000 7fef75865700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fef7019d7e0 con 0x7fef700ffcf0 2023-03-27T10:22:22.722 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.720+0000 7fef75865700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fef7019d970 con 0x7fef70101870 2023-03-27T10:22:22.722 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.720+0000 7fef6e7fc700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fef70100ee0 0x7fef70196060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:22.722 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.720+0000 7fef6e7fc700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fef70100ee0 0x7fef70196060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45504/0 (socket says 172.21.15.159:45504) 2023-03-27T10:22:22.723 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.720+0000 7fef6e7fc700 1 -- 172.21.15.159:0/505495783 learned_addr learned my addr 172.21.15.159:0/505495783 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:22.723 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.721+0000 7fef6effd700 1 --2- 172.21.15.159:0/505495783 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fef700ffcf0 0x7fef70195b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:22.723 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.721+0000 7fef6f7fe700 1 --2- 172.21.15.159:0/505495783 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fef70101870 0x7fef7019d0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:22.723 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.721+0000 7fef6e7fc700 1 -- 172.21.15.159:0/505495783 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fef70101870 msgr2=0x7fef7019d0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:22.723 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.721+0000 7fef6e7fc700 1 --2- 172.21.15.159:0/505495783 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fef70101870 0x7fef7019d0d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:22.723 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.721+0000 7fef6e7fc700 1 -- 172.21.15.159:0/505495783 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fef700ffcf0 msgr2=0x7fef70195b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:22.723 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.721+0000 7fef6e7fc700 1 --2- 172.21.15.159:0/505495783 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fef700ffcf0 0x7fef70195b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:22.723 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.721+0000 7fef6e7fc700 1 -- 172.21.15.159:0/505495783 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fef68009c40 con 0x7fef70100ee0 2023-03-27T10:22:22.724 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.721+0000 7fef6f7fe700 1 --2- 172.21.15.159:0/505495783 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fef70101870 0x7fef7019d0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:22:22.724 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.721+0000 7fef6e7fc700 1 --2- 172.21.15.159:0/505495783 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fef70100ee0 0x7fef70196060 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fef6400e9c0 tx=0x7fef6400ed80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:22.724 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.721+0000 7fef74863700 1 -- 172.21.15.159:0/505495783 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef6400cbd0 con 0x7fef70100ee0 2023-03-27T10:22:22.724 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.721+0000 7fef75865700 1 -- 172.21.15.159:0/505495783 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fef7010b660 con 0x7fef70100ee0 2023-03-27T10:22:22.724 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.721+0000 7fef74863700 1 -- 172.21.15.159:0/505495783 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fef6400cd30 con 0x7fef70100ee0 2023-03-27T10:22:22.724 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.721+0000 7fef74863700 1 -- 172.21.15.159:0/505495783 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef64005580 con 0x7fef70100ee0 2023-03-27T10:22:22.724 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.721+0000 7fef75865700 1 -- 172.21.15.159:0/505495783 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fef7010b4d0 con 0x7fef70100ee0 2023-03-27T10:22:22.725 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.722+0000 7fef75865700 1 -- 172.21.15.159:0/505495783 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fef70106ba0 con 0x7fef70100ee0 2023-03-27T10:22:22.725 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.723+0000 7fef74863700 1 -- 172.21.15.159:0/505495783 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7fef6401e020 con 0x7fef70100ee0 2023-03-27T10:22:22.725 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.724+0000 7fef74863700 1 --2- 172.21.15.159:0/505495783 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fef5809b3a0 0x7fef5809d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:22.726 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.724+0000 7fef6effd700 1 --2- 172.21.15.159:0/505495783 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fef5809b3a0 0x7fef5809d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:22.726 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.724+0000 7fef74863700 1 -- 172.21.15.159:0/505495783 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7fef64014070 con 0x7fef70100ee0 2023-03-27T10:22:22.726 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.724+0000 7fef6effd700 1 --2- 172.21.15.159:0/505495783 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fef5809b3a0 0x7fef5809d850 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7fef60005950 tx=0x7fef600058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:22.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:22 smithi171 ceph-mon[94808]: pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:22.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:22 smithi171 ceph-mon[94808]: Deploying daemon mds.a on smithi159 2023-03-27T10:22:22.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:22 smithi171 ceph-mon[94808]: mds.? [v2:172.21.15.159:6838/793152328,v1:172.21.15.159:6839/793152328] up:boot 2023-03-27T10:22:22.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:22 smithi171 ceph-mon[94808]: fsmap 10 up:standby 2023-03-27T10:22:22.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:22 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2023-03-27T10:22:22.727 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:22.728+0000 7fef74863700 1 -- 172.21.15.159:0/505495783 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fef64090b30 con 0x7fef70100ee0 2023-03-27T10:22:22.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:22 smithi161 ceph-mon[94333]: pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:22.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:22 smithi161 ceph-mon[94333]: Deploying daemon mds.a on smithi159 2023-03-27T10:22:22.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:22 smithi161 ceph-mon[94333]: mds.? [v2:172.21.15.159:6838/793152328,v1:172.21.15.159:6839/793152328] up:boot 2023-03-27T10:22:22.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:22 smithi161 ceph-mon[94333]: fsmap 10 up:standby 2023-03-27T10:22:22.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:22 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2023-03-27T10:22:23.002 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:23.002+0000 7fef75865700 1 -- 172.21.15.159:0/505495783 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fef7002cdb0 con 0x7fef70100ee0 2023-03-27T10:22:23.003 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:23.003+0000 7fef74863700 1 -- 172.21.15.159:0/505495783 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 11 v11) v1 ==== 94+0+8094 (secure 0 0 0) 0x7fef64094150 con 0x7fef70100ee0 2023-03-27T10:22:23.003 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:23.003 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":11,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14556,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6835/3048624466","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6834","nonce":3048624466},{"type":"v1","addr":"172.21.15.159:6835","nonce":3048624466}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14562,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6837/1998956589","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6836","nonce":1998956589},{"type":"v1","addr":"172.21.15.159:6837","nonce":1998956589}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14619,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6839/793152328","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6838","nonce":793152328},{"type":"v1","addr":"172.21.15.159:6839","nonce":793152328}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":24424,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6833/1022778932","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6832","nonce":1022778932},{"type":"v1","addr":"172.21.15.171:6833","nonce":1022778932}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24440,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6833/2969583384","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6832","nonce":2969583384},{"type":"v1","addr":"172.21.15.161:6833","nonce":2969583384}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24464,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6835/761787339","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6834","nonce":761787339},{"type":"v1","addr":"172.21.15.161:6835","nonce":761787339}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24472,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6835/1823903399","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6834","nonce":1823903399},{"type":"v1","addr":"172.21.15.171:6835","nonce":1823903399}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24484,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6837/2737069329","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6836","nonce":2737069329},{"type":"v1","addr":"172.21.15.171:6837","nonce":2737069329}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24491,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6837/406989937","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6836","nonce":406989937},{"type":"v1","addr":"172.21.15.161:6837","nonce":406989937}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9},{"gid":24502,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6839/1538859602","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6838","nonce":1538859602},{"type":"v1","addr":"172.21.15.171:6839","nonce":1538859602}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":10}],"filesystems":[]} 2023-03-27T10:22:23.006 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:23.006+0000 7fef75865700 1 -- 172.21.15.159:0/505495783 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fef5809b3a0 msgr2=0x7fef5809d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:23.006 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:23.006+0000 7fef75865700 1 --2- 172.21.15.159:0/505495783 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fef5809b3a0 0x7fef5809d850 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7fef60005950 tx=0x7fef600058e0 comp rx=0 tx=0).stop 2023-03-27T10:22:23.006 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:23.006+0000 7fef75865700 1 -- 172.21.15.159:0/505495783 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fef70100ee0 msgr2=0x7fef70196060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:23.006 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:23.006+0000 7fef75865700 1 --2- 172.21.15.159:0/505495783 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fef70100ee0 0x7fef70196060 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fef6400e9c0 tx=0x7fef6400ed80 comp rx=0 tx=0).stop 2023-03-27T10:22:23.006 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:23.006+0000 7fef75865700 1 -- 172.21.15.159:0/505495783 shutdown_connections 2023-03-27T10:22:23.007 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:23.006+0000 7fef75865700 1 --2- 172.21.15.159:0/505495783 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fef5809b3a0 0x7fef5809d850 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:23.007 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:23.006+0000 7fef75865700 1 --2- 172.21.15.159:0/505495783 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fef70101870 0x7fef7019d0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:23.007 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:23.006+0000 7fef75865700 1 --2- 172.21.15.159:0/505495783 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fef700ffcf0 0x7fef70195b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:23.007 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:23.006+0000 7fef75865700 1 --2- 172.21.15.159:0/505495783 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fef70100ee0 0x7fef70196060 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:23.007 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:23.006+0000 7fef75865700 1 -- 172.21.15.159:0/505495783 >> 172.21.15.159:0/505495783 conn(0x7fef700fb2f0 msgr2=0x7fef700fd4f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:23.007 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:23.007+0000 7fef75865700 1 -- 172.21.15.159:0/505495783 shutdown_connections 2023-03-27T10:22:23.007 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:23.007+0000 7fef75865700 1 -- 172.21.15.159:0/505495783 wait complete. 2023-03-27T10:22:23.008 INFO:teuthology.orchestra.run.smithi159.stderr:dumped fsmap epoch 11 2023-03-27T10:22:23.193 INFO:journalctl@ceph.mds.a.smithi159.stdout:Mar 27 10:22:22 smithi159 systemd[1]: Starting Ceph mds.a for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:23.445 INFO:journalctl@ceph.mds.a.smithi159.stdout:Mar 27 10:22:23 smithi159 podman[127630]: 2023-03-27T10:22:23.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:23 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/505495783' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:23.833 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:23 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/505495783' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:23.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:23 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/505495783' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:23.865 INFO:journalctl@ceph.mds.a.smithi159.stdout:Mar 27 10:22:23 smithi159 ceph-mds[127670]: -- 172.21.15.159:0/4117362338 <== mon.0 v2:172.21.15.159:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55610c552b60 con 0x55610c528c00 2023-03-27T10:22:23.865 INFO:journalctl@ceph.mds.a.smithi159.stdout:Mar 27 10:22:23 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-a[127646]: starting mds.a at 2023-03-27T10:22:23.865 INFO:journalctl@ceph.mds.a.smithi159.stdout:Mar 27 10:22:23 smithi159 bash[127630]: 2c1811fe4096e974703fcbc0d47c41f889f38b6ec6a4a54822b0e0e262a045ca 2023-03-27T10:22:23.865 INFO:journalctl@ceph.mds.a.smithi159.stdout:Mar 27 10:22:23 smithi159 systemd[1]: Started Ceph mds.a for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:22:24.527 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:24 smithi161 ceph-mon[94333]: pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:24.527 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:24.528 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:24.528 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:24.528 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:24.528 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:24.528 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:24 smithi161 ceph-mon[94333]: Deploying daemon mds.b on smithi161 2023-03-27T10:22:24.528 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:24 smithi161 ceph-mon[94333]: mds.? [v2:172.21.15.159:6840/536144142,v1:172.21.15.159:6841/536144142] up:boot 2023-03-27T10:22:24.528 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:24 smithi161 ceph-mon[94333]: fsmap 11 up:standby 2023-03-27T10:22:24.529 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:24 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2023-03-27T10:22:24.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:24 smithi159 ceph-mon[93039]: pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:24.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:24.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:24.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:24.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:24.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:24.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:24 smithi159 ceph-mon[93039]: Deploying daemon mds.b on smithi161 2023-03-27T10:22:24.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:24 smithi159 ceph-mon[93039]: mds.? [v2:172.21.15.159:6840/536144142,v1:172.21.15.159:6841/536144142] up:boot 2023-03-27T10:22:24.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:24 smithi159 ceph-mon[93039]: fsmap 11 up:standby 2023-03-27T10:22:24.866 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:24 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2023-03-27T10:22:24.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:24 smithi171 ceph-mon[94808]: pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:24.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:24.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:24.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-03-27T10:22:24.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-03-27T10:22:24.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:24.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:24 smithi171 ceph-mon[94808]: Deploying daemon mds.b on smithi161 2023-03-27T10:22:24.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:24 smithi171 ceph-mon[94808]: mds.? [v2:172.21.15.159:6840/536144142,v1:172.21.15.159:6841/536144142] up:boot 2023-03-27T10:22:24.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:24 smithi171 ceph-mon[94808]: fsmap 11 up:standby 2023-03-27T10:22:24.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:24 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2023-03-27T10:22:25.022 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-03-27T10:22:25.139 INFO:journalctl@ceph.mds.b.smithi161.stdout:Mar 27 10:22:25 smithi161 systemd[1]: Starting Ceph mds.b for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:25.172 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.173+0000 7f799e939700 1 -- 172.21.15.159:0/433253140 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7998103120 msgr2=0x7f7998105500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:25.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.173+0000 7f799e939700 1 --2- 172.21.15.159:0/433253140 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7998103120 0x7f7998105500 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f7988009990 tx=0x7f7988009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:25.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.173+0000 7f799e939700 1 -- 172.21.15.159:0/433253140 shutdown_connections 2023-03-27T10:22:25.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.173+0000 7f799e939700 1 --2- 172.21.15.159:0/433253140 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7998105a40 0x7f7998107e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:25.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.173+0000 7f799e939700 1 --2- 172.21.15.159:0/433253140 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7998100800 0x7f7998102be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:25.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.173+0000 7f799e939700 1 --2- 172.21.15.159:0/433253140 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7998103120 0x7f7998105500 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:25.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.173+0000 7f799e939700 1 -- 172.21.15.159:0/433253140 >> 172.21.15.159:0/433253140 conn(0x7f79980fa1e0 msgr2=0x7f79980fc650 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:25.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.173+0000 7f799e939700 1 -- 172.21.15.159:0/433253140 shutdown_connections 2023-03-27T10:22:25.174 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.173+0000 7f799e939700 1 -- 172.21.15.159:0/433253140 wait complete. 2023-03-27T10:22:25.175 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.173+0000 7f799e939700 1 Processor -- start 2023-03-27T10:22:25.175 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f799e939700 1 -- start start 2023-03-27T10:22:25.175 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f799e939700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7998100800 0x7f7998197c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:25.175 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f799e939700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7998103120 0x7f79981981c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:25.175 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f799e939700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7998105a40 0x7f799819f1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:25.176 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f799e939700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f799819f7c0 con 0x7f7998105a40 2023-03-27T10:22:25.176 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f799e939700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f799819f930 con 0x7f7998103120 2023-03-27T10:22:25.176 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f799e939700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f799819faf0 con 0x7f7998100800 2023-03-27T10:22:25.176 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f799ced6700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7998105a40 0x7f799819f1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:25.176 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f79977fe700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7998103120 0x7f79981981c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:25.176 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f799ced6700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7998105a40 0x7f799819f1f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45534/0 (socket says 172.21.15.159:45534) 2023-03-27T10:22:25.176 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f799ced6700 1 -- 172.21.15.159:0/1998231808 learned_addr learned my addr 172.21.15.159:0/1998231808 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:25.177 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f79977fe700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7998103120 0x7f79981981c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.161:3300/0 says I am v2:172.21.15.159:50724/0 (socket says 172.21.15.159:50724) 2023-03-27T10:22:25.177 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f7997fff700 1 --2- 172.21.15.159:0/1998231808 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7998100800 0x7f7998197c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:25.177 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f799ced6700 1 -- 172.21.15.159:0/1998231808 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7998100800 msgr2=0x7f7998197c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:25.177 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f799ced6700 1 --2- 172.21.15.159:0/1998231808 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7998100800 0x7f7998197c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:25.177 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f799ced6700 1 -- 172.21.15.159:0/1998231808 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7998103120 msgr2=0x7f79981981c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:25.177 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f799ced6700 1 --2- 172.21.15.159:0/1998231808 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7998103120 0x7f79981981c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:25.177 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.174+0000 7f799ced6700 1 -- 172.21.15.159:0/1998231808 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f79880095f0 con 0x7f7998105a40 2023-03-27T10:22:25.177 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.175+0000 7f799ced6700 1 --2- 172.21.15.159:0/1998231808 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7998105a40 0x7f799819f1f0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f799000ada0 tx=0x7f799000c330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:25.178 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.175+0000 7f79957fa700 1 -- 172.21.15.159:0/1998231808 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f799000f070 con 0x7f7998105a40 2023-03-27T10:22:25.178 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.175+0000 7f79957fa700 1 -- 172.21.15.159:0/1998231808 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7990007b30 con 0x7f7998105a40 2023-03-27T10:22:25.178 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.175+0000 7f799e939700 1 -- 172.21.15.159:0/1998231808 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f799810b640 con 0x7f7998105a40 2023-03-27T10:22:25.178 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.175+0000 7f79957fa700 1 -- 172.21.15.159:0/1998231808 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7990005280 con 0x7f7998105a40 2023-03-27T10:22:25.178 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.175+0000 7f799e939700 1 -- 172.21.15.159:0/1998231808 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f799810b4b0 con 0x7f7998105a40 2023-03-27T10:22:25.178 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.176+0000 7f799e939700 1 -- 172.21.15.159:0/1998231808 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f79981075b0 con 0x7f7998105a40 2023-03-27T10:22:25.179 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.177+0000 7f79957fa700 1 -- 172.21.15.159:0/1998231808 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f7990019020 con 0x7f7998105a40 2023-03-27T10:22:25.179 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.177+0000 7f79957fa700 1 --2- 172.21.15.159:0/1998231808 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f798409b3a0 0x7f798409d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:25.179 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.177+0000 7f7997fff700 1 --2- 172.21.15.159:0/1998231808 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f798409b3a0 0x7f798409d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:25.179 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.177+0000 7f79957fa700 1 -- 172.21.15.159:0/1998231808 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f79900c43f0 con 0x7f7998105a40 2023-03-27T10:22:25.180 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.178+0000 7f7997fff700 1 --2- 172.21.15.159:0/1998231808 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f798409b3a0 0x7f798409d850 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f7998199150 tx=0x7f798000b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:25.180 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.181+0000 7f79957fa700 1 -- 172.21.15.159:0/1998231808 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f7990090290 con 0x7f7998105a40 2023-03-27T10:22:25.406 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.406+0000 7f799e939700 1 -- 172.21.15.159:0/1998231808 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f799802cdb0 con 0x7f7998105a40 2023-03-27T10:22:25.406 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.407+0000 7f79957fa700 1 -- 172.21.15.159:0/1998231808 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 12 v12) v1 ==== 94+0+8852 (secure 0 0 0) 0x7f79900938b0 con 0x7f7998105a40 2023-03-27T10:22:25.406 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:25.407 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":12,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14556,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6835/3048624466","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6834","nonce":3048624466},{"type":"v1","addr":"172.21.15.159:6835","nonce":3048624466}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14562,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6837/1998956589","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6836","nonce":1998956589},{"type":"v1","addr":"172.21.15.159:6837","nonce":1998956589}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14619,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6839/793152328","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6838","nonce":793152328},{"type":"v1","addr":"172.21.15.159:6839","nonce":793152328}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14631,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6841/536144142","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6840","nonce":536144142},{"type":"v1","addr":"172.21.15.159:6841","nonce":536144142}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24424,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6833/1022778932","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6832","nonce":1022778932},{"type":"v1","addr":"172.21.15.171:6833","nonce":1022778932}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24440,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6833/2969583384","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6832","nonce":2969583384},{"type":"v1","addr":"172.21.15.161:6833","nonce":2969583384}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24464,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6835/761787339","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6834","nonce":761787339},{"type":"v1","addr":"172.21.15.161:6835","nonce":761787339}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24472,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6835/1823903399","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6834","nonce":1823903399},{"type":"v1","addr":"172.21.15.171:6835","nonce":1823903399}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24484,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6837/2737069329","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6836","nonce":2737069329},{"type":"v1","addr":"172.21.15.171:6837","nonce":2737069329}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24491,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6837/406989937","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6836","nonce":406989937},{"type":"v1","addr":"172.21.15.161:6837","nonce":406989937}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9},{"gid":24502,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6839/1538859602","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6838","nonce":1538859602},{"type":"v1","addr":"172.21.15.171:6839","nonce":1538859602}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":10}],"filesystems":[]} 2023-03-27T10:22:25.408 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.409+0000 7f799e939700 1 -- 172.21.15.159:0/1998231808 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f798409b3a0 msgr2=0x7f798409d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:25.408 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.409+0000 7f799e939700 1 --2- 172.21.15.159:0/1998231808 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f798409b3a0 0x7f798409d850 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f7998199150 tx=0x7f798000b410 comp rx=0 tx=0).stop 2023-03-27T10:22:25.409 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.409+0000 7f799e939700 1 -- 172.21.15.159:0/1998231808 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7998105a40 msgr2=0x7f799819f1f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:25.409 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.409+0000 7f799e939700 1 --2- 172.21.15.159:0/1998231808 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7998105a40 0x7f799819f1f0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f799000ada0 tx=0x7f799000c330 comp rx=0 tx=0).stop 2023-03-27T10:22:25.409 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.409+0000 7f799e939700 1 -- 172.21.15.159:0/1998231808 shutdown_connections 2023-03-27T10:22:25.409 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.409+0000 7f799e939700 1 --2- 172.21.15.159:0/1998231808 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f798409b3a0 0x7f798409d850 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:25.409 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.409+0000 7f799e939700 1 --2- 172.21.15.159:0/1998231808 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7998100800 0x7f7998197c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:25.409 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.409+0000 7f799e939700 1 --2- 172.21.15.159:0/1998231808 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7998105a40 0x7f799819f1f0 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:25.409 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.409+0000 7f799e939700 1 --2- 172.21.15.159:0/1998231808 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7998103120 0x7f79981981c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:25.410 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.409+0000 7f799e939700 1 -- 172.21.15.159:0/1998231808 >> 172.21.15.159:0/1998231808 conn(0x7f79980fa1e0 msgr2=0x7f79980fee80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:25.410 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.409+0000 7f799e939700 1 -- 172.21.15.159:0/1998231808 shutdown_connections 2023-03-27T10:22:25.410 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:25.409+0000 7f799e939700 1 -- 172.21.15.159:0/1998231808 wait complete. 2023-03-27T10:22:25.410 INFO:teuthology.orchestra.run.smithi159.stderr:dumped fsmap epoch 12 2023-03-27T10:22:25.735 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:25 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/1998231808' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:25.735 INFO:journalctl@ceph.mds.b.smithi161.stdout:Mar 27 10:22:25 smithi161 podman[119416]: 2023-03-27T10:22:25.735 INFO:journalctl@ceph.mds.b.smithi161.stdout:Mar 27 10:22:25 smithi161 ceph-mds[119456]: -- 172.21.15.161:0/3810593385 <== mon.2 v2:172.21.15.161:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5589cfe3cb60 con 0x5589cfe13000 2023-03-27T10:22:25.736 INFO:journalctl@ceph.mds.b.smithi161.stdout:Mar 27 10:22:25 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-b[119432]: starting mds.b at 2023-03-27T10:22:25.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:25 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/1998231808' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:25.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:25 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/1998231808' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:26.083 INFO:journalctl@ceph.mds.b.smithi161.stdout:Mar 27 10:22:25 smithi161 bash[119416]: 6c438d8893b25d9dc4df9ec98c843bd1b7cd06d323509c28dbaf0a8d495558e8 2023-03-27T10:22:26.084 INFO:journalctl@ceph.mds.b.smithi161.stdout:Mar 27 10:22:25 smithi161 systemd[1]: Started Ceph mds.b for 2c3cd140-cc88-11ed-9afe-001a4aab830c. 2023-03-27T10:22:26.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:26 smithi161 ceph-mon[94333]: pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:26.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:26 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:26.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:26 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:26.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:26 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:22:26.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:26 smithi159 ceph-mon[93039]: pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:26.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:26 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:26.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:26 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:26.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:26 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:22:26.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:26 smithi171 ceph-mon[94808]: pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:26.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:26 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:26.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:26 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:26.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:26 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:22:27.422 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-03-27T10:22:27.548 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.549+0000 7f1cdbada700 1 -- 172.21.15.159:0/3101125882 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1cd40747b0 msgr2=0x7f1cd4072c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:27.549 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.549+0000 7f1cdbada700 1 --2- 172.21.15.159:0/3101125882 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1cd40747b0 0x7f1cd4072c00 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f1cd0009990 tx=0x7f1cd0009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:27.549 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.549+0000 7f1cdbada700 1 -- 172.21.15.159:0/3101125882 shutdown_connections 2023-03-27T10:22:27.549 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.549+0000 7f1cdbada700 1 --2- 172.21.15.159:0/3101125882 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1cd4107b80 0x7f1cd410a000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:27.549 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.549+0000 7f1cdbada700 1 --2- 172.21.15.159:0/3101125882 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1cd40747b0 0x7f1cd4072c00 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:27.549 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.549+0000 7f1cdbada700 1 --2- 172.21.15.159:0/3101125882 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1cd40731d0 0x7f1cd4073640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:27.549 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.549+0000 7f1cdbada700 1 -- 172.21.15.159:0/3101125882 >> 172.21.15.159:0/3101125882 conn(0x7f1cd40fb2f0 msgr2=0x7f1cd40fd740 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:27.550 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.549+0000 7f1cdbada700 1 -- 172.21.15.159:0/3101125882 shutdown_connections 2023-03-27T10:22:27.550 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.549+0000 7f1cdbada700 1 -- 172.21.15.159:0/3101125882 wait complete. 2023-03-27T10:22:27.551 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cdbada700 1 Processor -- start 2023-03-27T10:22:27.551 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cdbada700 1 -- start start 2023-03-27T10:22:27.552 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cdbada700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1cd40731d0 0x7f1cd4197c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:27.552 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cdbada700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1cd40747b0 0x7f1cd41981c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:27.552 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cdbada700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1cd4107b80 0x7f1cd419f1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:27.552 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cdbada700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1cd419f7c0 con 0x7f1cd40731d0 2023-03-27T10:22:27.552 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cdbada700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1cd419f930 con 0x7f1cd4107b80 2023-03-27T10:22:27.552 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cdbada700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1cd419faf0 con 0x7f1cd40747b0 2023-03-27T10:22:27.552 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cd9876700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1cd40731d0 0x7f1cd4197c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:27.552 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cd9876700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1cd40731d0 0x7f1cd4197c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45540/0 (socket says 172.21.15.159:45540) 2023-03-27T10:22:27.553 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cd9876700 1 -- 172.21.15.159:0/830999709 learned_addr learned my addr 172.21.15.159:0/830999709 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:27.553 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cda077700 1 --2- 172.21.15.159:0/830999709 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1cd4107b80 0x7f1cd419f1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:27.553 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cd9075700 1 --2- 172.21.15.159:0/830999709 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1cd40747b0 0x7f1cd41981c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:27.553 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cd9876700 1 -- 172.21.15.159:0/830999709 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1cd40747b0 msgr2=0x7f1cd41981c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:27.553 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cd9876700 1 --2- 172.21.15.159:0/830999709 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1cd40747b0 0x7f1cd41981c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:27.553 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cd9876700 1 -- 172.21.15.159:0/830999709 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1cd4107b80 msgr2=0x7f1cd419f1f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:27.553 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cd9876700 1 --2- 172.21.15.159:0/830999709 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1cd4107b80 0x7f1cd419f1f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:27.554 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cd9876700 1 -- 172.21.15.159:0/830999709 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1cd00095f0 con 0x7f1cd40731d0 2023-03-27T10:22:27.554 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.550+0000 7f1cda077700 1 --2- 172.21.15.159:0/830999709 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1cd4107b80 0x7f1cd419f1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-03-27T10:22:27.554 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.551+0000 7f1cd9876700 1 --2- 172.21.15.159:0/830999709 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1cd40731d0 0x7f1cd4197c80 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f1cd00059d0 tx=0x7f1cd0004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:27.554 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.551+0000 7f1ccaffd700 1 -- 172.21.15.159:0/830999709 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cd001d070 con 0x7f1cd40731d0 2023-03-27T10:22:27.554 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.551+0000 7f1ccaffd700 1 -- 172.21.15.159:0/830999709 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1cd000fb90 con 0x7f1cd40731d0 2023-03-27T10:22:27.554 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.551+0000 7f1cdbada700 1 -- 172.21.15.159:0/830999709 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1cd410d830 con 0x7f1cd40731d0 2023-03-27T10:22:27.554 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.551+0000 7f1ccaffd700 1 -- 172.21.15.159:0/830999709 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cd00173f0 con 0x7f1cd40731d0 2023-03-27T10:22:27.554 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.551+0000 7f1cdbada700 1 -- 172.21.15.159:0/830999709 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1cd410d6a0 con 0x7f1cd40731d0 2023-03-27T10:22:27.555 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.552+0000 7f1cdbada700 1 -- 172.21.15.159:0/830999709 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1cd41062c0 con 0x7f1cd40731d0 2023-03-27T10:22:27.555 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.553+0000 7f1ccaffd700 1 -- 172.21.15.159:0/830999709 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f1cd0029030 con 0x7f1cd40731d0 2023-03-27T10:22:27.555 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.553+0000 7f1ccaffd700 1 --2- 172.21.15.159:0/830999709 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1cc009b3a0 0x7f1cc009d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:27.556 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.554+0000 7f1cd9075700 1 --2- 172.21.15.159:0/830999709 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1cc009b3a0 0x7f1cc009d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:27.556 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.554+0000 7f1ccaffd700 1 -- 172.21.15.159:0/830999709 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f1cd00c5d60 con 0x7f1cd40731d0 2023-03-27T10:22:27.556 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.554+0000 7f1cd9075700 1 --2- 172.21.15.159:0/830999709 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1cc009b3a0 0x7f1cc009d850 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f1cc4009c60 tx=0x7f1cc4009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:27.557 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.558+0000 7f1ccaffd700 1 -- 172.21.15.159:0/830999709 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f1cd005b960 con 0x7f1cd40731d0 2023-03-27T10:22:27.810 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.811+0000 7f1cdbada700 1 -- 172.21.15.159:0/830999709 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1cd402cdb0 con 0x7f1cd40731d0 2023-03-27T10:22:27.810 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.811+0000 7f1ccaffd700 1 -- 172.21.15.159:0/830999709 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 13 v13) v1 ==== 94+0+9613 (secure 0 0 0) 0x7f1cd0091ba0 con 0x7f1cd40731d0 2023-03-27T10:22:27.811 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:27.811 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":13,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14556,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6835/3048624466","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6834","nonce":3048624466},{"type":"v1","addr":"172.21.15.159:6835","nonce":3048624466}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14562,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6837/1998956589","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6836","nonce":1998956589},{"type":"v1","addr":"172.21.15.159:6837","nonce":1998956589}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14619,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6839/793152328","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6838","nonce":793152328},{"type":"v1","addr":"172.21.15.159:6839","nonce":793152328}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14631,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6841/536144142","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6840","nonce":536144142},{"type":"v1","addr":"172.21.15.159:6841","nonce":536144142}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24424,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6833/1022778932","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6832","nonce":1022778932},{"type":"v1","addr":"172.21.15.171:6833","nonce":1022778932}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24440,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6833/2969583384","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6832","nonce":2969583384},{"type":"v1","addr":"172.21.15.161:6833","nonce":2969583384}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24464,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6835/761787339","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6834","nonce":761787339},{"type":"v1","addr":"172.21.15.161:6835","nonce":761787339}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24472,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6835/1823903399","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6834","nonce":1823903399},{"type":"v1","addr":"172.21.15.171:6835","nonce":1823903399}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24484,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6837/2737069329","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6836","nonce":2737069329},{"type":"v1","addr":"172.21.15.171:6837","nonce":2737069329}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24491,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6837/406989937","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6836","nonce":406989937},{"type":"v1","addr":"172.21.15.161:6837","nonce":406989937}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9},{"gid":24502,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6839/1538859602","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6838","nonce":1538859602},{"type":"v1","addr":"172.21.15.171:6839","nonce":1538859602}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":10},{"gid":24539,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6839/2491878460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6838","nonce":2491878460},{"type":"v1","addr":"172.21.15.161:6839","nonce":2491878460}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":13}],"filesystems":[]} 2023-03-27T10:22:27.813 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.813+0000 7f1cdbada700 1 -- 172.21.15.159:0/830999709 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1cc009b3a0 msgr2=0x7f1cc009d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:27.813 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.813+0000 7f1cdbada700 1 --2- 172.21.15.159:0/830999709 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1cc009b3a0 0x7f1cc009d850 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f1cc4009c60 tx=0x7f1cc4009260 comp rx=0 tx=0).stop 2023-03-27T10:22:27.813 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.814+0000 7f1cdbada700 1 -- 172.21.15.159:0/830999709 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1cd40731d0 msgr2=0x7f1cd4197c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:27.813 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.814+0000 7f1cdbada700 1 --2- 172.21.15.159:0/830999709 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1cd40731d0 0x7f1cd4197c80 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f1cd00059d0 tx=0x7f1cd0004730 comp rx=0 tx=0).stop 2023-03-27T10:22:27.813 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.814+0000 7f1cdbada700 1 -- 172.21.15.159:0/830999709 shutdown_connections 2023-03-27T10:22:27.814 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.814+0000 7f1cdbada700 1 --2- 172.21.15.159:0/830999709 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1cc009b3a0 0x7f1cc009d850 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:27.814 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.814+0000 7f1cdbada700 1 --2- 172.21.15.159:0/830999709 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1cd4107b80 0x7f1cd419f1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:27.814 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.814+0000 7f1cdbada700 1 --2- 172.21.15.159:0/830999709 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1cd40731d0 0x7f1cd4197c80 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:27.814 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.814+0000 7f1cdbada700 1 --2- 172.21.15.159:0/830999709 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1cd40747b0 0x7f1cd41981c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:27.814 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.814+0000 7f1cdbada700 1 -- 172.21.15.159:0/830999709 >> 172.21.15.159:0/830999709 conn(0x7f1cd40fb2f0 msgr2=0x7f1cd40fd6d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:27.814 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.814+0000 7f1cdbada700 1 -- 172.21.15.159:0/830999709 shutdown_connections 2023-03-27T10:22:27.814 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.814+0000 7f1cdbada700 1 -- 172.21.15.159:0/830999709 wait complete. 2023-03-27T10:22:27.815 INFO:teuthology.orchestra.run.smithi159.stderr:dumped fsmap epoch 13 2023-03-27T10:22:27.824 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:27 smithi159 ceph-mon[93039]: mds.? [v2:172.21.15.161:6838/2491878460,v1:172.21.15.161:6839/2491878460] up:boot 2023-03-27T10:22:27.824 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:27 smithi159 ceph-mon[93039]: fsmap 12 up:standby 2023-03-27T10:22:27.824 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:27 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2023-03-27T10:22:27.824 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:27 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:27.825 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-03-27T10:22:27.833 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:27 smithi161 ceph-mon[94333]: mds.? [v2:172.21.15.161:6838/2491878460,v1:172.21.15.161:6839/2491878460] up:boot 2023-03-27T10:22:27.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:27 smithi161 ceph-mon[94333]: fsmap 12 up:standby 2023-03-27T10:22:27.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:27 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2023-03-27T10:22:27.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:27 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:27.958 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.959+0000 7f713c0f6700 1 -- 172.21.15.159:0/3414172207 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7134107ac0 msgr2=0x7f7134109f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:27.959 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.959+0000 7f713c0f6700 1 --2- 172.21.15.159:0/3414172207 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7134107ac0 0x7f7134109f70 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f712c00b0a0 tx=0x7f712c00b3b0 comp rx=0 tx=0).stop 2023-03-27T10:22:27.959 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.959+0000 7f713c0f6700 1 -- 172.21.15.159:0/3414172207 shutdown_connections 2023-03-27T10:22:27.959 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.959+0000 7f713c0f6700 1 --2- 172.21.15.159:0/3414172207 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7134107ac0 0x7f7134109f70 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:27.959 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.959+0000 7f713c0f6700 1 --2- 172.21.15.159:0/3414172207 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7134068600 0x7f7134104c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:27.959 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.959+0000 7f713c0f6700 1 --2- 172.21.15.159:0/3414172207 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f71341051a0 0x7f7134107580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:27.959 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.959+0000 7f713c0f6700 1 -- 172.21.15.159:0/3414172207 >> 172.21.15.159:0/3414172207 conn(0x7f71340fa210 msgr2=0x7f71340fc620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:27.960 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.959+0000 7f713c0f6700 1 -- 172.21.15.159:0/3414172207 shutdown_connections 2023-03-27T10:22:27.960 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.959+0000 7f713c0f6700 1 -- 172.21.15.159:0/3414172207 wait complete. 2023-03-27T10:22:27.961 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f713c0f6700 1 Processor -- start 2023-03-27T10:22:27.961 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f713c0f6700 1 -- start start 2023-03-27T10:22:27.962 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f713c0f6700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7134068600 0x7f7134197dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:27.962 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f713c0f6700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f71341051a0 0x7f7134198300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:27.962 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f713c0f6700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7134107ac0 0x7f713419f370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:27.962 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f713c0f6700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f713419f940 con 0x7f71341051a0 2023-03-27T10:22:27.962 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f713c0f6700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f713419fa80 con 0x7f7134068600 2023-03-27T10:22:27.962 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f713c0f6700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f713419fc10 con 0x7f7134107ac0 2023-03-27T10:22:27.962 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f7139691700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f71341051a0 0x7f7134198300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:27.963 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f7139691700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f71341051a0 0x7f7134198300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45554/0 (socket says 172.21.15.159:45554) 2023-03-27T10:22:27.963 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f7139691700 1 -- 172.21.15.159:0/4125505438 learned_addr learned my addr 172.21.15.159:0/4125505438 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:27.963 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f713a693700 1 --2- 172.21.15.159:0/4125505438 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7134107ac0 0x7f713419f370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:27.963 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f7139e92700 1 --2- 172.21.15.159:0/4125505438 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7134068600 0x7f7134197dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:27.963 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f7139691700 1 -- 172.21.15.159:0/4125505438 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7134107ac0 msgr2=0x7f713419f370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:27.963 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f7139691700 1 --2- 172.21.15.159:0/4125505438 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7134107ac0 0x7f713419f370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:27.963 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f7139691700 1 -- 172.21.15.159:0/4125505438 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7134068600 msgr2=0x7f7134197dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:27.963 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f7139691700 1 --2- 172.21.15.159:0/4125505438 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7134068600 0x7f7134197dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:27.964 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f7139691700 1 -- 172.21.15.159:0/4125505438 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f712c009d00 con 0x7f71341051a0 2023-03-27T10:22:27.964 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.960+0000 7f7139691700 1 --2- 172.21.15.159:0/4125505438 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f71341051a0 0x7f7134198300 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f712800e9a0 tx=0x7f712800ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:27.964 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.961+0000 7f7126ffd700 1 -- 172.21.15.159:0/4125505438 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7128009790 con 0x7f71341051a0 2023-03-27T10:22:27.964 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.961+0000 7f7126ffd700 1 -- 172.21.15.159:0/4125505438 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f712800cbd0 con 0x7f71341051a0 2023-03-27T10:22:27.964 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.961+0000 7f7126ffd700 1 -- 172.21.15.159:0/4125505438 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7128018680 con 0x7f71341051a0 2023-03-27T10:22:27.964 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.961+0000 7f713c0f6700 1 -- 172.21.15.159:0/4125505438 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f713410d850 con 0x7f71341051a0 2023-03-27T10:22:27.964 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.961+0000 7f713c0f6700 1 -- 172.21.15.159:0/4125505438 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f713410d6c0 con 0x7f71341051a0 2023-03-27T10:22:27.964 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.962+0000 7f713c0f6700 1 -- 172.21.15.159:0/4125505438 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f71340fbd80 con 0x7f71341051a0 2023-03-27T10:22:27.965 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.963+0000 7f7126ffd700 1 -- 172.21.15.159:0/4125505438 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f712800c040 con 0x7f71341051a0 2023-03-27T10:22:27.965 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.964+0000 7f7126ffd700 1 --2- 172.21.15.159:0/4125505438 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f712009b420 0x7f712009d8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:27.965 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.964+0000 7f7139e92700 1 --2- 172.21.15.159:0/4125505438 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f712009b420 0x7f712009d8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:27.966 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.964+0000 7f7126ffd700 1 -- 172.21.15.159:0/4125505438 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f7128014070 con 0x7f71341051a0 2023-03-27T10:22:27.966 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.964+0000 7f7139e92700 1 --2- 172.21.15.159:0/4125505438 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f712009b420 0x7f712009d8d0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f7134068320 tx=0x7f713000b5b0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:27.966 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:27.967+0000 7f7126ffd700 1 -- 172.21.15.159:0/4125505438 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f7128090f10 con 0x7f71341051a0 2023-03-27T10:22:27.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:27 smithi171 ceph-mon[94808]: mds.? [v2:172.21.15.161:6838/2491878460,v1:172.21.15.161:6839/2491878460] up:boot 2023-03-27T10:22:27.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:27 smithi171 ceph-mon[94808]: fsmap 12 up:standby 2023-03-27T10:22:27.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:27 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2023-03-27T10:22:27.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:27 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:28.168 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.169+0000 7f713c0f6700 1 -- 172.21.15.159:0/4125505438 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f71340613e0 con 0x7f71341051a0 2023-03-27T10:22:28.169 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.170+0000 7f7126ffd700 1 -- 172.21.15.159:0/4125505438 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v81) v1 ==== 74+0+18455 (secure 0 0 0) 0x7f7128094530 con 0x7f71341051a0 2023-03-27T10:22:28.170 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:28.170 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":81,"fsid":"2c3cd140-cc88-11ed-9afe-001a4aab830c","created":"2023-03-27T10:15:32.617230+0000","modified":"2023-03-27T10:21:50.939009+0000","last_up_change":"2023-03-27T10:21:49.442480+0000","last_in_change":"2023-03-27T10:21:32.265278+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-03-27T10:18:35.277335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f47f99b9-47f9-4d94-83d9-33b1278be17e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6803","nonce":87273908}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6805","nonce":87273908}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6809","nonce":87273908}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6807","nonce":87273908}]},"public_addr":"172.21.15.159:6803/87273908","cluster_addr":"172.21.15.159:6805/87273908","heartbeat_back_addr":"172.21.15.159:6809/87273908","heartbeat_front_addr":"172.21.15.159:6807/87273908","state":["exists","up"]},{"osd":1,"uuid":"97f66da9-22b4-4eb5-9c77-9d37cf0fca82","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6800","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6801","nonce":2094166586}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6802","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6803","nonce":2094166586}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6806","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6807","nonce":2094166586}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6804","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6805","nonce":2094166586}]},"public_addr":"172.21.15.161:6801/2094166586","cluster_addr":"172.21.15.161:6803/2094166586","heartbeat_back_addr":"172.21.15.161:6807/2094166586","heartbeat_front_addr":"172.21.15.161:6805/2094166586","state":["exists","up"]},{"osd":2,"uuid":"ba9f5f82-3aef-42f7-a9e6-0e07aa7d8df1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6801","nonce":2724827688}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6803","nonce":2724827688}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6807","nonce":2724827688}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6805","nonce":2724827688}]},"public_addr":"172.21.15.171:6801/2724827688","cluster_addr":"172.21.15.171:6803/2724827688","heartbeat_back_addr":"172.21.15.171:6807/2724827688","heartbeat_front_addr":"172.21.15.171:6805/2724827688","state":["exists","up"]},{"osd":3,"uuid":"4ef4b519-fdef-4d38-b387-bc14ea381bd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6811","nonce":1992276871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6813","nonce":1992276871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6817","nonce":1992276871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6815","nonce":1992276871}]},"public_addr":"172.21.15.159:6811/1992276871","cluster_addr":"172.21.15.159:6813/1992276871","heartbeat_back_addr":"172.21.15.159:6817/1992276871","heartbeat_front_addr":"172.21.15.159:6815/1992276871","state":["exists","up"]},{"osd":4,"uuid":"982675c1-dd87-46b4-97ef-a5dde1b1d777","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6808","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6809","nonce":3978602555}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6810","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6811","nonce":3978602555}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6814","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6815","nonce":3978602555}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6812","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6813","nonce":3978602555}]},"public_addr":"172.21.15.161:6809/3978602555","cluster_addr":"172.21.15.161:6811/3978602555","heartbeat_back_addr":"172.21.15.161:6815/3978602555","heartbeat_front_addr":"172.21.15.161:6813/3978602555","state":["exists","up"]},{"osd":5,"uuid":"b4de4a85-4f91-4194-a1ce-db12f1898c4f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6809","nonce":569611891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6811","nonce":569611891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6815","nonce":569611891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6813","nonce":569611891}]},"public_addr":"172.21.15.171:6809/569611891","cluster_addr":"172.21.15.171:6811/569611891","heartbeat_back_addr":"172.21.15.171:6815/569611891","heartbeat_front_addr":"172.21.15.171:6813/569611891","state":["exists","up"]},{"osd":6,"uuid":"dff7d5f4-655f-4baa-9084-03591a4b3bb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6819","nonce":1176978350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6821","nonce":1176978350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6825","nonce":1176978350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6823","nonce":1176978350}]},"public_addr":"172.21.15.159:6819/1176978350","cluster_addr":"172.21.15.159:6821/1176978350","heartbeat_back_addr":"172.21.15.159:6825/1176978350","heartbeat_front_addr":"172.21.15.159:6823/1176978350","state":["exists","up"]},{"osd":7,"uuid":"30f026d9-6963-4468-95e8-78c7ed2e89f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6816","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6817","nonce":2098174471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6818","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6819","nonce":2098174471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6822","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6823","nonce":2098174471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6820","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6821","nonce":2098174471}]},"public_addr":"172.21.15.161:6817/2098174471","cluster_addr":"172.21.15.161:6819/2098174471","heartbeat_back_addr":"172.21.15.161:6823/2098174471","heartbeat_front_addr":"172.21.15.161:6821/2098174471","state":["exists","up"]},{"osd":8,"uuid":"9f69fbcc-32ff-4d4b-b562-ee0a12745cc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6817","nonce":1033586906}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6819","nonce":1033586906}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6823","nonce":1033586906}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6821","nonce":1033586906}]},"public_addr":"172.21.15.171:6817/1033586906","cluster_addr":"172.21.15.171:6819/1033586906","heartbeat_back_addr":"172.21.15.171:6823/1033586906","heartbeat_front_addr":"172.21.15.171:6821/1033586906","state":["exists","up"]},{"osd":9,"uuid":"d7fb9eed-901b-4943-9529-6edcba2385e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6827","nonce":3285179829}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6829","nonce":3285179829}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6832","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6833","nonce":3285179829}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6831","nonce":3285179829}]},"public_addr":"172.21.15.159:6827/3285179829","cluster_addr":"172.21.15.159:6829/3285179829","heartbeat_back_addr":"172.21.15.159:6833/3285179829","heartbeat_front_addr":"172.21.15.159:6831/3285179829","state":["exists","up"]},{"osd":10,"uuid":"4871ca03-30ae-4e87-942c-9441dd31fb40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6824","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6825","nonce":946677932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6826","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6827","nonce":946677932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6830","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6831","nonce":946677932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6828","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6829","nonce":946677932}]},"public_addr":"172.21.15.161:6825/946677932","cluster_addr":"172.21.15.161:6827/946677932","heartbeat_back_addr":"172.21.15.161:6831/946677932","heartbeat_front_addr":"172.21.15.161:6829/946677932","state":["exists","up"]},{"osd":11,"uuid":"1fb37d9a-ed52-4c0d-a082-65e17d242106","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6825","nonce":596725174}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6827","nonce":596725174}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6831","nonce":596725174}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6829","nonce":596725174}]},"public_addr":"172.21.15.171:6825/596725174","cluster_addr":"172.21.15.171:6827/596725174","heartbeat_back_addr":"172.21.15.171:6831/596725174","heartbeat_front_addr":"172.21.15.171:6829/596725174","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:17:49.128592+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:10.944593+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:32.989432+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:54.675294+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:16.543485+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:38.390326+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:59.566202+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:20:21.348420+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:20:43.383576+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:04.710992+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:26.286573+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:48.277317+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.159:0/653469093":"2023-03-28T10:16:26.900382+0000","172.21.15.159:6800/3561797228":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/3045586257":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/2859804517":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6801/3561797228":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/3487813906":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/1306320900":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6800/1729835725":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6801/1729835725":"2023-03-28T10:16:04.737100+0000","172.21.15.159:0/72206061":"2023-03-28T10:16:04.737100+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-03-27T10:22:28.172 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.173+0000 7f713c0f6700 1 -- 172.21.15.159:0/4125505438 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f712009b420 msgr2=0x7f712009d8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:28.172 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.173+0000 7f713c0f6700 1 --2- 172.21.15.159:0/4125505438 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f712009b420 0x7f712009d8d0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f7134068320 tx=0x7f713000b5b0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.172 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.173+0000 7f713c0f6700 1 -- 172.21.15.159:0/4125505438 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f71341051a0 msgr2=0x7f7134198300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:28.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.173+0000 7f713c0f6700 1 --2- 172.21.15.159:0/4125505438 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f71341051a0 0x7f7134198300 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f712800e9a0 tx=0x7f712800ed60 comp rx=0 tx=0).stop 2023-03-27T10:22:28.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.173+0000 7f713c0f6700 1 -- 172.21.15.159:0/4125505438 shutdown_connections 2023-03-27T10:22:28.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.173+0000 7f713c0f6700 1 --2- 172.21.15.159:0/4125505438 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f712009b420 0x7f712009d8d0 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.173+0000 7f713c0f6700 1 --2- 172.21.15.159:0/4125505438 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7134107ac0 0x7f713419f370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.173+0000 7f713c0f6700 1 --2- 172.21.15.159:0/4125505438 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7134068600 0x7f7134197dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.173+0000 7f713c0f6700 1 --2- 172.21.15.159:0/4125505438 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f71341051a0 0x7f7134198300 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.173 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.173+0000 7f713c0f6700 1 -- 172.21.15.159:0/4125505438 >> 172.21.15.159:0/4125505438 conn(0x7f71340fa210 msgr2=0x7f7134105de0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:28.174 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.173+0000 7f713c0f6700 1 -- 172.21.15.159:0/4125505438 shutdown_connections 2023-03-27T10:22:28.174 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.173+0000 7f713c0f6700 1 -- 172.21.15.159:0/4125505438 wait complete. 2023-03-27T10:22:28.187 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-03-27T10:18:35.277335+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2023-03-27T10:22:28.187 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2023-03-27T10:22:28.313 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.314+0000 7f724e521700 1 -- 172.21.15.159:0/1995411823 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f72480685c0 msgr2=0x7f72481028a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:28.314 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.314+0000 7f724e521700 1 --2- 172.21.15.159:0/1995411823 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f72480685c0 0x7f72481028a0 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f7238009990 tx=0x7f7238009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.314 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.314+0000 7f724e521700 1 -- 172.21.15.159:0/1995411823 shutdown_connections 2023-03-27T10:22:28.314 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.314+0000 7f724e521700 1 --2- 172.21.15.159:0/1995411823 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f72481058b0 0x7f7248107d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.314 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.314+0000 7f724e521700 1 --2- 172.21.15.159:0/1995411823 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f72480685c0 0x7f72481028a0 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.314 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.314+0000 7f724e521700 1 --2- 172.21.15.159:0/1995411823 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f7248102de0 0x7f7248105220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.314 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.314+0000 7f724e521700 1 -- 172.21.15.159:0/1995411823 >> 172.21.15.159:0/1995411823 conn(0x7f72480fa210 msgr2=0x7f72480fc620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:28.315 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.314+0000 7f724e521700 1 -- 172.21.15.159:0/1995411823 shutdown_connections 2023-03-27T10:22:28.315 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.314+0000 7f724e521700 1 -- 172.21.15.159:0/1995411823 wait complete. 2023-03-27T10:22:28.315 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.314+0000 7f724e521700 1 Processor -- start 2023-03-27T10:22:28.315 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.314+0000 7f724e521700 1 -- start start 2023-03-27T10:22:28.316 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f724e521700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f72480685c0 0x7f7248195a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:28.317 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f724e521700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7248102de0 0x7f7248195fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:28.317 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f724e521700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f72481058b0 0x7f724819d040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:28.317 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f724e521700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f724819d610 con 0x7f7248102de0 2023-03-27T10:22:28.317 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f724e521700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f724819d750 con 0x7f72481058b0 2023-03-27T10:22:28.317 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f724e521700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f724819d8e0 con 0x7f72480685c0 2023-03-27T10:22:28.317 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f72477fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7248102de0 0x7f7248195fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:28.317 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f72477fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7248102de0 0x7f7248195fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45566/0 (socket says 172.21.15.159:45566) 2023-03-27T10:22:28.318 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f72477fe700 1 -- 172.21.15.159:0/1924277811 learned_addr learned my addr 172.21.15.159:0/1924277811 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:28.318 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f724cabe700 1 --2- 172.21.15.159:0/1924277811 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f72481058b0 0x7f724819d040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:28.318 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f7247fff700 1 --2- 172.21.15.159:0/1924277811 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f72480685c0 0x7f7248195a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:28.318 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f72477fe700 1 -- 172.21.15.159:0/1924277811 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f72480685c0 msgr2=0x7f7248195a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:28.318 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f72477fe700 1 --2- 172.21.15.159:0/1924277811 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f72480685c0 0x7f7248195a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.318 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f72477fe700 1 -- 172.21.15.159:0/1924277811 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f72481058b0 msgr2=0x7f724819d040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:28.318 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f72477fe700 1 --2- 172.21.15.159:0/1924277811 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f72481058b0 0x7f724819d040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.319 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f72477fe700 1 -- 172.21.15.159:0/1924277811 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f72380095f0 con 0x7f7248102de0 2023-03-27T10:22:28.319 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f724cabe700 1 --2- 172.21.15.159:0/1924277811 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f72481058b0 0x7f724819d040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-03-27T10:22:28.319 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.315+0000 7f72477fe700 1 --2- 172.21.15.159:0/1924277811 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7248102de0 0x7f7248195fd0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f723c00d950 tx=0x7f723c00dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:28.319 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.316+0000 7f72457fa700 1 -- 172.21.15.159:0/1924277811 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f723c00f890 con 0x7f7248102de0 2023-03-27T10:22:28.319 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.316+0000 7f72457fa700 1 -- 172.21.15.159:0/1924277811 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f723c010460 con 0x7f7248102de0 2023-03-27T10:22:28.319 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.316+0000 7f724e521700 1 -- 172.21.15.159:0/1924277811 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f724810b640 con 0x7f7248102de0 2023-03-27T10:22:28.319 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.316+0000 7f72457fa700 1 -- 172.21.15.159:0/1924277811 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f723c00e610 con 0x7f7248102de0 2023-03-27T10:22:28.319 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.316+0000 7f724e521700 1 -- 172.21.15.159:0/1924277811 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f724810b4b0 con 0x7f7248102de0 2023-03-27T10:22:28.320 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.317+0000 7f724e521700 1 -- 172.21.15.159:0/1924277811 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f72480fbd80 con 0x7f7248102de0 2023-03-27T10:22:28.320 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.318+0000 7f72457fa700 1 -- 172.21.15.159:0/1924277811 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f723c022070 con 0x7f7248102de0 2023-03-27T10:22:28.320 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.318+0000 7f72457fa700 1 --2- 172.21.15.159:0/1924277811 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f723009b3a0 0x7f723009d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:28.320 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.318+0000 7f7247fff700 1 --2- 172.21.15.159:0/1924277811 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f723009b3a0 0x7f723009d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:28.321 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.318+0000 7f72457fa700 1 -- 172.21.15.159:0/1924277811 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f723c0c46e0 con 0x7f7248102de0 2023-03-27T10:22:28.321 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.319+0000 7f7247fff700 1 --2- 172.21.15.159:0/1924277811 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f723009b3a0 0x7f723009d850 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f723800b5c0 tx=0x7f7238005fd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:28.322 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.323+0000 7f72457fa700 1 -- 172.21.15.159:0/1924277811 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f723c0904f0 con 0x7f7248102de0 2023-03-27T10:22:28.519 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.520+0000 7f724e521700 1 -- 172.21.15.159:0/1924277811 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f72480613e0 con 0x7f7248102de0 2023-03-27T10:22:28.520 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.521+0000 7f72457fa700 1 -- 172.21.15.159:0/1924277811 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v81) v1 ==== 93+0+10 (secure 0 0 0) 0x7f723c020020 con 0x7f7248102de0 2023-03-27T10:22:28.520 INFO:teuthology.orchestra.run.smithi159.stdout:pg_num: 1 2023-03-27T10:22:28.522 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.523+0000 7f724e521700 1 -- 172.21.15.159:0/1924277811 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f723009b3a0 msgr2=0x7f723009d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:28.522 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.523+0000 7f724e521700 1 --2- 172.21.15.159:0/1924277811 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f723009b3a0 0x7f723009d850 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f723800b5c0 tx=0x7f7238005fd0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.522 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.523+0000 7f724e521700 1 -- 172.21.15.159:0/1924277811 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7248102de0 msgr2=0x7f7248195fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:28.522 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.523+0000 7f724e521700 1 --2- 172.21.15.159:0/1924277811 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7248102de0 0x7f7248195fd0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f723c00d950 tx=0x7f723c00dc60 comp rx=0 tx=0).stop 2023-03-27T10:22:28.523 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.523+0000 7f724e521700 1 -- 172.21.15.159:0/1924277811 shutdown_connections 2023-03-27T10:22:28.523 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.523+0000 7f724e521700 1 --2- 172.21.15.159:0/1924277811 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f723009b3a0 0x7f723009d850 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.523 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.523+0000 7f724e521700 1 --2- 172.21.15.159:0/1924277811 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f72480685c0 0x7f7248195a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.523 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.523+0000 7f724e521700 1 --2- 172.21.15.159:0/1924277811 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f72481058b0 0x7f724819d040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.523 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.523+0000 7f724e521700 1 --2- 172.21.15.159:0/1924277811 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7248102de0 0x7f7248195fd0 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.523 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.523+0000 7f724e521700 1 -- 172.21.15.159:0/1924277811 >> 172.21.15.159:0/1924277811 conn(0x7f72480fa210 msgr2=0x7f7248101100 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:28.523 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.523+0000 7f724e521700 1 -- 172.21.15.159:0/1924277811 shutdown_connections 2023-03-27T10:22:28.524 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.523+0000 7f724e521700 1 -- 172.21.15.159:0/1924277811 wait complete. 2023-03-27T10:22:28.534 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2023-03-27T10:22:28.670 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.670+0000 7f1b5e807700 1 -- 172.21.15.159:0/1975032095 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1b58100780 msgr2=0x7f1b58107040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:28.670 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.670+0000 7f1b5e807700 1 --2- 172.21.15.159:0/1975032095 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1b58100780 0x7f1b58107040 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f1b5000b0a0 tx=0x7f1b5000b3b0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.670 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.670+0000 7f1b5e807700 1 -- 172.21.15.159:0/1975032095 shutdown_connections 2023-03-27T10:22:28.670 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.670+0000 7f1b5e807700 1 --2- 172.21.15.159:0/1975032095 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1b580fec00 0x7f1b580ff010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.670 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.670+0000 7f1b5e807700 1 --2- 172.21.15.159:0/1975032095 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1b58100780 0x7f1b58107040 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.671 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.670+0000 7f1b5e807700 1 --2- 172.21.15.159:0/1975032095 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1b580ffdf0 0x7f1b58100240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.671 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.670+0000 7f1b5e807700 1 -- 172.21.15.159:0/1975032095 >> 172.21.15.159:0/1975032095 conn(0x7f1b580fa1e0 msgr2=0x7f1b580fc650 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:28.671 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.670+0000 7f1b5e807700 1 -- 172.21.15.159:0/1975032095 shutdown_connections 2023-03-27T10:22:28.671 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.670+0000 7f1b5e807700 1 -- 172.21.15.159:0/1975032095 wait complete. 2023-03-27T10:22:28.672 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.671+0000 7f1b5e807700 1 Processor -- start 2023-03-27T10:22:28.673 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.671+0000 7f1b5e807700 1 -- start start 2023-03-27T10:22:28.673 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b5e807700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1b580fec00 0x7f1b58195b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:28.673 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b5e807700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1b580ffdf0 0x7f1b58196070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:28.673 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b5e807700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1b58100780 0x7f1b5819d0e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:28.673 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b5e807700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b5819d6b0 con 0x7f1b580ffdf0 2023-03-27T10:22:28.673 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b5e807700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b5819d7f0 con 0x7f1b580fec00 2023-03-27T10:22:28.674 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b5e807700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b5819d980 con 0x7f1b58100780 2023-03-27T10:22:28.674 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b577fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1b580ffdf0 0x7f1b58196070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:28.674 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b5cda4700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1b58100780 0x7f1b5819d0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:28.674 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b57fff700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1b580fec00 0x7f1b58195b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:28.674 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b577fe700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1b580ffdf0 0x7f1b58196070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45580/0 (socket says 172.21.15.159:45580) 2023-03-27T10:22:28.674 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b577fe700 1 -- 172.21.15.159:0/3987695746 learned_addr learned my addr 172.21.15.159:0/3987695746 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:28.674 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b5cda4700 1 -- 172.21.15.159:0/3987695746 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1b580fec00 msgr2=0x7f1b58195b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:28.674 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b5cda4700 1 --2- 172.21.15.159:0/3987695746 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1b580fec00 0x7f1b58195b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.675 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b5cda4700 1 -- 172.21.15.159:0/3987695746 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1b580ffdf0 msgr2=0x7f1b58196070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:28.675 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b5cda4700 1 --2- 172.21.15.159:0/3987695746 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1b580ffdf0 0x7f1b58196070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.675 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b5cda4700 1 -- 172.21.15.159:0/3987695746 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1b48009520 con 0x7f1b58100780 2023-03-27T10:22:28.675 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b5cda4700 1 --2- 172.21.15.159:0/3987695746 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1b58100780 0x7f1b5819d0e0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f1b5000b0a0 tx=0x7f1b50007f30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:28.675 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b557fa700 1 -- 172.21.15.159:0/3987695746 <== mon.1 v2:172.21.15.171:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b5000e040 con 0x7f1b58100780 2023-03-27T10:22:28.675 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b557fa700 1 -- 172.21.15.159:0/3987695746 <== mon.1 v2:172.21.15.171:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1b5000bde0 con 0x7f1b58100780 2023-03-27T10:22:28.675 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.672+0000 7f1b5e807700 1 -- 172.21.15.159:0/3987695746 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1b50009d00 con 0x7f1b58100780 2023-03-27T10:22:28.675 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.673+0000 7f1b557fa700 1 -- 172.21.15.159:0/3987695746 <== mon.1 v2:172.21.15.171:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b5001b6b0 con 0x7f1b58100780 2023-03-27T10:22:28.676 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.673+0000 7f1b5e807700 1 -- 172.21.15.159:0/3987695746 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1b5810a570 con 0x7f1b58100780 2023-03-27T10:22:28.676 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.673+0000 7f1b5e807700 1 -- 172.21.15.159:0/3987695746 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1b5810a700 con 0x7f1b58100780 2023-03-27T10:22:28.676 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.674+0000 7f1b557fa700 1 -- 172.21.15.159:0/3987695746 <== mon.1 v2:172.21.15.171:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f1b5001d480 con 0x7f1b58100780 2023-03-27T10:22:28.677 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.675+0000 7f1b557fa700 1 --2- 172.21.15.159:0/3987695746 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1b4009b3a0 0x7f1b4009d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:28.677 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.675+0000 7f1b57fff700 1 --2- 172.21.15.159:0/3987695746 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1b4009b3a0 0x7f1b4009d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:28.677 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.675+0000 7f1b557fa700 1 -- 172.21.15.159:0/3987695746 <== mon.1 v2:172.21.15.171:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f1b500c61a0 con 0x7f1b58100780 2023-03-27T10:22:28.677 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.675+0000 7f1b57fff700 1 --2- 172.21.15.159:0/3987695746 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1b4009b3a0 0x7f1b4009d850 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f1b580ffc50 tx=0x7f1b48009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:28.678 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.679+0000 7f1b557fa700 1 -- 172.21.15.159:0/3987695746 <== mon.1 v2:172.21.15.171:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f1b50092040 con 0x7f1b58100780 2023-03-27T10:22:28.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:28 smithi161 ceph-mon[94333]: pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:28.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:28 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/830999709' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:28.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:28 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/4125505438' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:22:28.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:28 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/1924277811' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-03-27T10:22:28.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:28 smithi159 ceph-mon[93039]: pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:28.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:28 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/830999709' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:28.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:28 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/4125505438' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:22:28.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:28 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/1924277811' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-03-27T10:22:28.882 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.883+0000 7f1b5e807700 1 -- 172.21.15.159:0/3987695746 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7f1b5819e3a0 con 0x7f1b58100780 2023-03-27T10:22:28.883 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.884+0000 7f1b557fa700 1 -- 172.21.15.159:0/3987695746 <== mon.1 v2:172.21.15.171:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v81) v1 ==== 84+0+65 (secure 0 0 0) 0x7f1b50095660 con 0x7f1b58100780 2023-03-27T10:22:28.883 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:28.884 INFO:teuthology.orchestra.run.smithi159.stdout:[ 2023-03-27T10:22:28.884 INFO:teuthology.orchestra.run.smithi159.stdout: { 2023-03-27T10:22:28.884 INFO:teuthology.orchestra.run.smithi159.stdout: "poolnum": 1, 2023-03-27T10:22:28.884 INFO:teuthology.orchestra.run.smithi159.stdout: "poolname": ".mgr" 2023-03-27T10:22:28.884 INFO:teuthology.orchestra.run.smithi159.stdout: } 2023-03-27T10:22:28.884 INFO:teuthology.orchestra.run.smithi159.stdout:] 2023-03-27T10:22:28.886 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.886+0000 7f1b5e807700 1 -- 172.21.15.159:0/3987695746 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1b4009b3a0 msgr2=0x7f1b4009d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:28.886 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.886+0000 7f1b5e807700 1 --2- 172.21.15.159:0/3987695746 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1b4009b3a0 0x7f1b4009d850 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f1b580ffc50 tx=0x7f1b48009260 comp rx=0 tx=0).stop 2023-03-27T10:22:28.886 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.886+0000 7f1b5e807700 1 -- 172.21.15.159:0/3987695746 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1b58100780 msgr2=0x7f1b5819d0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:28.886 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.886+0000 7f1b5e807700 1 --2- 172.21.15.159:0/3987695746 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1b58100780 0x7f1b5819d0e0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f1b5000b0a0 tx=0x7f1b50007f30 comp rx=0 tx=0).stop 2023-03-27T10:22:28.886 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.886+0000 7f1b5e807700 1 -- 172.21.15.159:0/3987695746 shutdown_connections 2023-03-27T10:22:28.887 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.887+0000 7f1b5e807700 1 --2- 172.21.15.159:0/3987695746 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1b4009b3a0 0x7f1b4009d850 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.887 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.887+0000 7f1b5e807700 1 --2- 172.21.15.159:0/3987695746 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1b58100780 0x7f1b5819d0e0 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.887 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.887+0000 7f1b5e807700 1 --2- 172.21.15.159:0/3987695746 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1b580fec00 0x7f1b58195b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.887 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.887+0000 7f1b5e807700 1 --2- 172.21.15.159:0/3987695746 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1b580ffdf0 0x7f1b58196070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:28.887 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.887+0000 7f1b5e807700 1 -- 172.21.15.159:0/3987695746 >> 172.21.15.159:0/3987695746 conn(0x7f1b580fa1e0 msgr2=0x7f1b58068960 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:28.887 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.887+0000 7f1b5e807700 1 -- 172.21.15.159:0/3987695746 shutdown_connections 2023-03-27T10:22:28.887 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:28.887+0000 7f1b5e807700 1 -- 172.21.15.159:0/3987695746 wait complete. 2023-03-27T10:22:28.898 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2023-03-27T10:22:28.899 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2023-03-27T10:22:28.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:28 smithi171 ceph-mon[94808]: pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:28.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:28 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/830999709' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:28.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:28 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/4125505438' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:22:28.976 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:28 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/1924277811' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-03-27T10:22:29.031 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.031+0000 7fc9b17c2700 1 -- 172.21.15.159:0/476198186 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc9ac074810 msgr2=0x7fc9ac072c60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:29.031 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.031+0000 7fc9b17c2700 1 --2- 172.21.15.159:0/476198186 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc9ac074810 0x7fc9ac072c60 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7fc99c009990 tx=0x7fc99c009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.031 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.031+0000 7fc9b17c2700 1 -- 172.21.15.159:0/476198186 shutdown_connections 2023-03-27T10:22:29.031 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.031+0000 7fc9b17c2700 1 --2- 172.21.15.159:0/476198186 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fc9ac107bb0 0x7fc9ac10a030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.031 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.031+0000 7fc9b17c2700 1 --2- 172.21.15.159:0/476198186 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc9ac074810 0x7fc9ac072c60 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.031 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.031+0000 7fc9b17c2700 1 --2- 172.21.15.159:0/476198186 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fc9ac073230 0x7fc9ac0736a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.032 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.031+0000 7fc9b17c2700 1 -- 172.21.15.159:0/476198186 >> 172.21.15.159:0/476198186 conn(0x7fc9ac0fb330 msgr2=0x7fc9ac0fd740 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:29.032 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.031+0000 7fc9b17c2700 1 -- 172.21.15.159:0/476198186 shutdown_connections 2023-03-27T10:22:29.032 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.031+0000 7fc9b17c2700 1 -- 172.21.15.159:0/476198186 wait complete. 2023-03-27T10:22:29.034 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.032+0000 7fc9b17c2700 1 Processor -- start 2023-03-27T10:22:29.034 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.032+0000 7fc9b17c2700 1 -- start start 2023-03-27T10:22:29.034 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.032+0000 7fc9b17c2700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc9ac073230 0x7fc9ac197cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:29.034 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.032+0000 7fc9b17c2700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fc9ac074810 0x7fc9ac198230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:29.034 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.032+0000 7fc9b17c2700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fc9ac107bb0 0x7fc9ac19f260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:29.034 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.032+0000 7fc9b17c2700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9ac19f830 con 0x7fc9ac073230 2023-03-27T10:22:29.034 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.032+0000 7fc9b17c2700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9ac19f9a0 con 0x7fc9ac107bb0 2023-03-27T10:22:29.035 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.032+0000 7fc9b17c2700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9ac19fb60 con 0x7fc9ac074810 2023-03-27T10:22:29.035 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.032+0000 7fc9aaffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc9ac073230 0x7fc9ac197cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:29.035 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.032+0000 7fc9aaffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc9ac073230 0x7fc9ac197cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45590/0 (socket says 172.21.15.159:45590) 2023-03-27T10:22:29.035 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.032+0000 7fc9aaffd700 1 -- 172.21.15.159:0/392443734 learned_addr learned my addr 172.21.15.159:0/392443734 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:29.035 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.032+0000 7fc9aa7fc700 1 --2- 172.21.15.159:0/392443734 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fc9ac074810 0x7fc9ac198230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:29.035 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.032+0000 7fc9ab7fe700 1 --2- 172.21.15.159:0/392443734 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fc9ac107bb0 0x7fc9ac19f260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:29.035 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.033+0000 7fc9aa7fc700 1 -- 172.21.15.159:0/392443734 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fc9ac107bb0 msgr2=0x7fc9ac19f260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:29.035 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.033+0000 7fc9aa7fc700 1 --2- 172.21.15.159:0/392443734 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fc9ac107bb0 0x7fc9ac19f260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.036 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.033+0000 7fc9aa7fc700 1 -- 172.21.15.159:0/392443734 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc9ac073230 msgr2=0x7fc9ac197cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:29.036 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.033+0000 7fc9aa7fc700 1 --2- 172.21.15.159:0/392443734 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc9ac073230 0x7fc9ac197cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.036 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.033+0000 7fc9aa7fc700 1 -- 172.21.15.159:0/392443734 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc9a4009c40 con 0x7fc9ac074810 2023-03-27T10:22:29.036 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.033+0000 7fc9aaffd700 1 --2- 172.21.15.159:0/392443734 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc9ac073230 0x7fc9ac197cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-03-27T10:22:29.036 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.033+0000 7fc9aa7fc700 1 --2- 172.21.15.159:0/392443734 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fc9ac074810 0x7fc9ac198230 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fc9a000c940 tx=0x7fc9a000cc50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:29.036 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.033+0000 7fc993fff700 1 -- 172.21.15.159:0/392443734 <== mon.1 v2:172.21.15.171:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9a0007890 con 0x7fc9ac074810 2023-03-27T10:22:29.036 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.033+0000 7fc9b17c2700 1 -- 172.21.15.159:0/392443734 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc99c0095f0 con 0x7fc9ac074810 2023-03-27T10:22:29.037 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.033+0000 7fc9b17c2700 1 -- 172.21.15.159:0/392443734 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc9ac10d860 con 0x7fc9ac074810 2023-03-27T10:22:29.037 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.033+0000 7fc993fff700 1 -- 172.21.15.159:0/392443734 <== mon.1 v2:172.21.15.171:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc9a000a550 con 0x7fc9ac074810 2023-03-27T10:22:29.037 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.033+0000 7fc993fff700 1 -- 172.21.15.159:0/392443734 <== mon.1 v2:172.21.15.171:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9a000f450 con 0x7fc9ac074810 2023-03-27T10:22:29.037 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.034+0000 7fc9b17c2700 1 -- 172.21.15.159:0/392443734 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc9ac106260 con 0x7fc9ac074810 2023-03-27T10:22:29.038 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.035+0000 7fc993fff700 1 -- 172.21.15.159:0/392443734 <== mon.1 v2:172.21.15.171:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7fc9a0004030 con 0x7fc9ac074810 2023-03-27T10:22:29.038 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.035+0000 7fc993fff700 1 --2- 172.21.15.159:0/392443734 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fc99409b420 0x7fc99409d8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:29.038 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.036+0000 7fc9aaffd700 1 --2- 172.21.15.159:0/392443734 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fc99409b420 0x7fc99409d8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:29.038 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.036+0000 7fc993fff700 1 -- 172.21.15.159:0/392443734 <== mon.1 v2:172.21.15.171:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7127+0+0 (secure 0 0 0) 0x7fc9a0013070 con 0x7fc9ac074810 2023-03-27T10:22:29.038 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.036+0000 7fc9aaffd700 1 --2- 172.21.15.159:0/392443734 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fc99409b420 0x7fc99409d8d0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fc99c00b680 tx=0x7fc99c005fd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:29.039 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.040+0000 7fc993fff700 1 -- 172.21.15.159:0/392443734 <== mon.1 v2:172.21.15.171:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fc9a00cb080 con 0x7fc9ac074810 2023-03-27T10:22:29.235 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.236+0000 7fc9b17c2700 1 -- 172.21.15.159:0/392443734 --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7fc9ac061480 con 0x7fc9ac074810 2023-03-27T10:22:29.821 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.822+0000 7fc993fff700 1 -- 172.21.15.159:0/392443734 <== mon.1 v2:172.21.15.171:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v82) v1 ==== 138+0+0 (secure 0 0 0) 0x7fc9a0090c50 con 0x7fc9ac074810 2023-03-27T10:22:29.825 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.825+0000 7fc9b17c2700 1 -- 172.21.15.159:0/392443734 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fc99409b420 msgr2=0x7fc99409d8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:29.825 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.825+0000 7fc9b17c2700 1 --2- 172.21.15.159:0/392443734 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fc99409b420 0x7fc99409d8d0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fc99c00b680 tx=0x7fc99c005fd0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.825 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.825+0000 7fc9b17c2700 1 -- 172.21.15.159:0/392443734 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fc9ac074810 msgr2=0x7fc9ac198230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:29.825 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.825+0000 7fc9b17c2700 1 --2- 172.21.15.159:0/392443734 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fc9ac074810 0x7fc9ac198230 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fc9a000c940 tx=0x7fc9a000cc50 comp rx=0 tx=0).stop 2023-03-27T10:22:29.825 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.825+0000 7fc9b17c2700 1 -- 172.21.15.159:0/392443734 shutdown_connections 2023-03-27T10:22:29.825 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.825+0000 7fc9b17c2700 1 --2- 172.21.15.159:0/392443734 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fc99409b420 0x7fc99409d8d0 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.826 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.825+0000 7fc9b17c2700 1 --2- 172.21.15.159:0/392443734 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fc9ac107bb0 0x7fc9ac19f260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.826 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.825+0000 7fc9b17c2700 1 --2- 172.21.15.159:0/392443734 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc9ac073230 0x7fc9ac197cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.826 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.825+0000 7fc9b17c2700 1 --2- 172.21.15.159:0/392443734 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fc9ac074810 0x7fc9ac198230 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.826 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.825+0000 7fc9b17c2700 1 -- 172.21.15.159:0/392443734 >> 172.21.15.159:0/392443734 conn(0x7fc9ac0fb330 msgr2=0x7fc9ac0fd710 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:29.826 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.825+0000 7fc9b17c2700 1 -- 172.21.15.159:0/392443734 shutdown_connections 2023-03-27T10:22:29.826 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.825+0000 7fc9b17c2700 1 -- 172.21.15.159:0/392443734 wait complete. 2023-03-27T10:22:29.827 INFO:teuthology.orchestra.run.smithi159.stderr:pool 'cephfs_metadata' created 2023-03-27T10:22:29.837 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.05 2023-03-27T10:22:29.970 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.969+0000 7f9de3fea700 1 -- 172.21.15.159:0/1491585402 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9ddc0ff550 msgr2=0x7f9ddc0ff9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:29.971 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.969+0000 7f9de3fea700 1 --2- 172.21.15.159:0/1491585402 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9ddc0ff550 0x7f9ddc0ff9a0 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f9dcc009990 tx=0x7f9dcc009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.971 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.969+0000 7f9de3fea700 1 -- 172.21.15.159:0/1491585402 shutdown_connections 2023-03-27T10:22:29.971 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.969+0000 7f9de3fea700 1 --2- 172.21.15.159:0/1491585402 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f9ddc0fe360 0x7f9ddc0fe770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.971 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.969+0000 7f9de3fea700 1 --2- 172.21.15.159:0/1491585402 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f9ddc0ffee0 0x7f9ddc1067a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.971 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.969+0000 7f9de3fea700 1 --2- 172.21.15.159:0/1491585402 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9ddc0ff550 0x7f9ddc0ff9a0 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.971 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.969+0000 7f9de3fea700 1 -- 172.21.15.159:0/1491585402 >> 172.21.15.159:0/1491585402 conn(0x7f9ddc0f99a0 msgr2=0x7f9ddc0fbdb0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:29.971 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.970+0000 7f9de3fea700 1 -- 172.21.15.159:0/1491585402 shutdown_connections 2023-03-27T10:22:29.972 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.970+0000 7f9de3fea700 1 -- 172.21.15.159:0/1491585402 wait complete. 2023-03-27T10:22:29.972 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.970+0000 7f9de3fea700 1 Processor -- start 2023-03-27T10:22:29.972 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.970+0000 7f9de3fea700 1 -- start start 2023-03-27T10:22:29.972 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.970+0000 7f9de3fea700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f9ddc0fe360 0x7f9ddc1022f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:29.972 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.970+0000 7f9de3fea700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9ddc0ff550 0x7f9ddc100930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:29.972 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.970+0000 7f9de3fea700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f9ddc0ffee0 0x7f9ddc100e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:29.972 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.970+0000 7f9de3fea700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ddc101610 con 0x7f9ddc0ff550 2023-03-27T10:22:29.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.970+0000 7f9de3fea700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ddc101780 con 0x7f9ddc0ffee0 2023-03-27T10:22:29.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.970+0000 7f9de3fea700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ddc19fb00 con 0x7f9ddc0fe360 2023-03-27T10:22:29.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.970+0000 7f9de1585700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9ddc0ff550 0x7f9ddc100930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:29.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.970+0000 7f9de1585700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9ddc0ff550 0x7f9ddc100930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45606/0 (socket says 172.21.15.159:45606) 2023-03-27T10:22:29.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.970+0000 7f9de1585700 1 -- 172.21.15.159:0/4279510122 learned_addr learned my addr 172.21.15.159:0/4279510122 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:29.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.970+0000 7f9de1d86700 1 --2- 172.21.15.159:0/4279510122 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f9ddc0fe360 0x7f9ddc1022f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:29.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.971+0000 7f9de2587700 1 --2- 172.21.15.159:0/4279510122 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f9ddc0ffee0 0x7f9ddc100e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:29.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.971+0000 7f9de1585700 1 -- 172.21.15.159:0/4279510122 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f9ddc0fe360 msgr2=0x7f9ddc1022f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:29.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.971+0000 7f9de1585700 1 --2- 172.21.15.159:0/4279510122 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f9ddc0fe360 0x7f9ddc1022f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.971+0000 7f9de1585700 1 -- 172.21.15.159:0/4279510122 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f9ddc0ffee0 msgr2=0x7f9ddc100e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:29.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.971+0000 7f9de1585700 1 --2- 172.21.15.159:0/4279510122 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f9ddc0ffee0 0x7f9ddc100e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:29.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.971+0000 7f9de1585700 1 -- 172.21.15.159:0/4279510122 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9dcc0095f0 con 0x7f9ddc0ff550 2023-03-27T10:22:29.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.971+0000 7f9de1585700 1 --2- 172.21.15.159:0/4279510122 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9ddc0ff550 0x7f9ddc100930 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f9dcc000c00 tx=0x7f9dcc0047a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:29.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.971+0000 7f9dd2ffd700 1 -- 172.21.15.159:0/4279510122 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9dcc01d070 con 0x7f9ddc0ff550 2023-03-27T10:22:29.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.971+0000 7f9dd2ffd700 1 -- 172.21.15.159:0/4279510122 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9dcc022470 con 0x7f9ddc0ff550 2023-03-27T10:22:29.975 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.971+0000 7f9de3fea700 1 -- 172.21.15.159:0/4279510122 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9ddc109500 con 0x7f9ddc0ff550 2023-03-27T10:22:29.975 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.971+0000 7f9dd2ffd700 1 -- 172.21.15.159:0/4279510122 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9dcc017650 con 0x7f9ddc0ff550 2023-03-27T10:22:29.975 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.971+0000 7f9de3fea700 1 -- 172.21.15.159:0/4279510122 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9ddc109370 con 0x7f9ddc0ff550 2023-03-27T10:22:29.976 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.972+0000 7f9de3fea700 1 -- 172.21.15.159:0/4279510122 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9ddc1a0010 con 0x7f9ddc0ff550 2023-03-27T10:22:29.976 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.973+0000 7f9dd2ffd700 1 -- 172.21.15.159:0/4279510122 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f9dcc029030 con 0x7f9ddc0ff550 2023-03-27T10:22:29.976 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.974+0000 7f9dd2ffd700 1 --2- 172.21.15.159:0/4279510122 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f9dc809b3a0 0x7f9dc809d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:29.976 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.974+0000 7f9dd2ffd700 1 -- 172.21.15.159:0/4279510122 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(82..82 src has 1..82) v4 ==== 7497+0+0 (secure 0 0 0) 0x7f9dcc0c6460 con 0x7f9ddc0ff550 2023-03-27T10:22:29.976 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.974+0000 7f9de1d86700 1 --2- 172.21.15.159:0/4279510122 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f9dc809b3a0 0x7f9dc809d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:29.976 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.974+0000 7f9de1d86700 1 --2- 172.21.15.159:0/4279510122 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f9dc809b3a0 0x7f9dc809d850 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f9ddc0ff3b0 tx=0x7f9dd8010040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:29.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:29 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:29.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:29 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:29.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:29 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:29.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:29 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:29.977 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:29 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:29.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:29 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:29.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:29 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3987695746' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2023-03-27T10:22:29.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:29 smithi171 ceph-mon[94808]: pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:29.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:29 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/392443734' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2023-03-27T10:22:29.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:29 smithi171 ceph-mon[94808]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2023-03-27T10:22:29.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:29 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:29.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:29 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:22:29.978 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:29 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:29.979 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:29 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:22:29.979 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:29.977+0000 7f9dd2ffd700 1 -- 172.21.15.159:0/4279510122 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f9dcc092190 con 0x7f9ddc0ff550 2023-03-27T10:22:30.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:29 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:30.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:29 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:30.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:29 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:30.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:29 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:30.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:29 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:30.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:29 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:30.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:29 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3987695746' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2023-03-27T10:22:30.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:29 smithi161 ceph-mon[94333]: pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:30.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:29 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/392443734' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2023-03-27T10:22:30.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:29 smithi161 ceph-mon[94333]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2023-03-27T10:22:30.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:29 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:30.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:29 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:22:30.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:29 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:30.086 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:29 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:22:30.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:29 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:30.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:29 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:30.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:29 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:30.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:29 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:30.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:29 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:30.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:29 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:30.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:29 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3987695746' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2023-03-27T10:22:30.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:29 smithi159 ceph-mon[93039]: pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:30.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:29 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/392443734' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2023-03-27T10:22:30.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:29 smithi159 ceph-mon[93039]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2023-03-27T10:22:30.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:29 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:30.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:29 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:22:30.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:29 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:30.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:29 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-03-27T10:22:30.179 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.180+0000 7f9de3fea700 1 -- 172.21.15.159:0/4279510122 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05} v 0) v1 -- 0x7f9ddc1a05c0 con 0x7f9ddc0ff550 2023-03-27T10:22:30.833 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.834+0000 7f9dd2ffd700 1 -- 172.21.15.159:0/4279510122 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]=0 pool 'cephfs_data' created v83) v1 ==== 171+0+0 (secure 0 0 0) 0x7f9dcc0957b0 con 0x7f9ddc0ff550 2023-03-27T10:22:30.842 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.843+0000 7f9de3fea700 1 -- 172.21.15.159:0/4279510122 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f9dc809b3a0 msgr2=0x7f9dc809d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:30.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.843+0000 7f9de3fea700 1 --2- 172.21.15.159:0/4279510122 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f9dc809b3a0 0x7f9dc809d850 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f9ddc0ff3b0 tx=0x7f9dd8010040 comp rx=0 tx=0).stop 2023-03-27T10:22:30.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.843+0000 7f9de3fea700 1 -- 172.21.15.159:0/4279510122 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9ddc0ff550 msgr2=0x7f9ddc100930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:30.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.843+0000 7f9de3fea700 1 --2- 172.21.15.159:0/4279510122 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9ddc0ff550 0x7f9ddc100930 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f9dcc000c00 tx=0x7f9dcc0047a0 comp rx=0 tx=0).stop 2023-03-27T10:22:30.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.843+0000 7f9de3fea700 1 -- 172.21.15.159:0/4279510122 shutdown_connections 2023-03-27T10:22:30.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.843+0000 7f9de3fea700 1 --2- 172.21.15.159:0/4279510122 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f9dc809b3a0 0x7f9dc809d850 secure :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f9ddc0ff3b0 tx=0x7f9dd8010040 comp rx=0 tx=0).stop 2023-03-27T10:22:30.843 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.843+0000 7f9de3fea700 1 --2- 172.21.15.159:0/4279510122 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f9ddc0fe360 0x7f9ddc1022f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:30.844 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.843+0000 7f9de3fea700 1 --2- 172.21.15.159:0/4279510122 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f9ddc0ffee0 0x7f9ddc100e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:30.844 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.843+0000 7f9de3fea700 1 --2- 172.21.15.159:0/4279510122 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9ddc0ff550 0x7f9ddc100930 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:30.844 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.843+0000 7f9de3fea700 1 -- 172.21.15.159:0/4279510122 >> 172.21.15.159:0/4279510122 conn(0x7f9ddc0f99a0 msgr2=0x7f9ddc104c50 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:30.846 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.847+0000 7f9de3fea700 1 -- 172.21.15.159:0/4279510122 shutdown_connections 2023-03-27T10:22:30.848 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.849+0000 7f9de3fea700 1 -- 172.21.15.159:0/4279510122 wait complete. 2023-03-27T10:22:30.849 INFO:teuthology.orchestra.run.smithi159.stderr:pool 'cephfs_data' created 2023-03-27T10:22:30.862 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2023-03-27T10:22:30.989 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.990+0000 7fc03c78f700 1 -- 172.21.15.159:0/1219297836 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc0341030b0 msgr2=0x7fc034105490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:30.990 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.990+0000 7fc03c78f700 1 --2- 172.21.15.159:0/1219297836 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc0341030b0 0x7fc034105490 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7fc030009990 tx=0x7fc030009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:30.990 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.990+0000 7fc03c78f700 1 -- 172.21.15.159:0/1219297836 shutdown_connections 2023-03-27T10:22:30.990 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.990+0000 7fc03c78f700 1 --2- 172.21.15.159:0/1219297836 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fc0341059d0 0x7fc034107e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:30.990 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.990+0000 7fc03c78f700 1 --2- 172.21.15.159:0/1219297836 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fc034100790 0x7fc034102b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:30.990 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.990+0000 7fc03c78f700 1 --2- 172.21.15.159:0/1219297836 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc0341030b0 0x7fc034105490 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:30.990 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.990+0000 7fc03c78f700 1 -- 172.21.15.159:0/1219297836 >> 172.21.15.159:0/1219297836 conn(0x7fc0340fa230 msgr2=0x7fc0340fc660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:30.991 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.990+0000 7fc03c78f700 1 -- 172.21.15.159:0/1219297836 shutdown_connections 2023-03-27T10:22:30.991 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.990+0000 7fc03c78f700 1 -- 172.21.15.159:0/1219297836 wait complete. 2023-03-27T10:22:30.992 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.991+0000 7fc03c78f700 1 Processor -- start 2023-03-27T10:22:30.993 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.991+0000 7fc03c78f700 1 -- start start 2023-03-27T10:22:30.993 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.991+0000 7fc03c78f700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc034100790 0x7fc034195af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:30.993 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.991+0000 7fc03c78f700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fc0341030b0 0x7fc034196030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:30.993 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.991+0000 7fc03c78f700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fc0341059d0 0x7fc03419d0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:30.993 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.991+0000 7fc03c78f700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc03419d670 con 0x7fc034100790 2023-03-27T10:22:30.993 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.991+0000 7fc03c78f700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc03419d7b0 con 0x7fc0341030b0 2023-03-27T10:22:30.994 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.991+0000 7fc03c78f700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc03419d940 con 0x7fc0341059d0 2023-03-27T10:22:30.994 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.991+0000 7fc03a52b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc034100790 0x7fc034195af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:30.994 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.991+0000 7fc03a52b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc034100790 0x7fc034195af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45622/0 (socket says 172.21.15.159:45622) 2023-03-27T10:22:30.994 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.991+0000 7fc03a52b700 1 -- 172.21.15.159:0/3674937540 learned_addr learned my addr 172.21.15.159:0/3674937540 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:30.994 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.991+0000 7fc039d2a700 1 --2- 172.21.15.159:0/3674937540 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fc0341030b0 0x7fc034196030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:30.994 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.991+0000 7fc03a52b700 1 -- 172.21.15.159:0/3674937540 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fc0341059d0 msgr2=0x7fc03419d0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:30.994 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.991+0000 7fc03ad2c700 1 --2- 172.21.15.159:0/3674937540 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fc0341059d0 0x7fc03419d0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:30.994 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.991+0000 7fc03a52b700 1 --2- 172.21.15.159:0/3674937540 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fc0341059d0 0x7fc03419d0a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:30.995 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.992+0000 7fc03a52b700 1 -- 172.21.15.159:0/3674937540 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fc0341030b0 msgr2=0x7fc034196030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:30.995 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.992+0000 7fc03a52b700 1 --2- 172.21.15.159:0/3674937540 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fc0341030b0 0x7fc034196030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:30.995 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.992+0000 7fc03a52b700 1 -- 172.21.15.159:0/3674937540 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0300095f0 con 0x7fc034100790 2023-03-27T10:22:30.995 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.992+0000 7fc03a52b700 1 --2- 172.21.15.159:0/3674937540 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc034100790 0x7fc034195af0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7fc02400d950 tx=0x7fc02400dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:30.995 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.992+0000 7fc02b7fe700 1 -- 172.21.15.159:0/3674937540 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc024009940 con 0x7fc034100790 2023-03-27T10:22:30.995 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.992+0000 7fc02b7fe700 1 -- 172.21.15.159:0/3674937540 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc024010460 con 0x7fc034100790 2023-03-27T10:22:30.995 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.992+0000 7fc03c78f700 1 -- 172.21.15.159:0/3674937540 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc03410b720 con 0x7fc034100790 2023-03-27T10:22:30.995 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.992+0000 7fc02b7fe700 1 -- 172.21.15.159:0/3674937540 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc02400f620 con 0x7fc034100790 2023-03-27T10:22:30.996 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.992+0000 7fc03c78f700 1 -- 172.21.15.159:0/3674937540 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc03410b590 con 0x7fc034100790 2023-03-27T10:22:30.996 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.993+0000 7fc03c78f700 1 -- 172.21.15.159:0/3674937540 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc0340fbda0 con 0x7fc034100790 2023-03-27T10:22:30.996 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.994+0000 7fc02b7fe700 1 -- 172.21.15.159:0/3674937540 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7fc024022070 con 0x7fc034100790 2023-03-27T10:22:30.997 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.994+0000 7fc02b7fe700 1 --2- 172.21.15.159:0/3674937540 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fc02009b3a0 0x7fc02009d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:30.997 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.995+0000 7fc039d2a700 1 --2- 172.21.15.159:0/3674937540 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fc02009b3a0 0x7fc02009d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:30.997 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.995+0000 7fc02b7fe700 1 -- 172.21.15.159:0/3674937540 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 7879+0+0 (secure 0 0 0) 0x7fc0240c4a90 con 0x7fc034100790 2023-03-27T10:22:30.997 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.995+0000 7fc039d2a700 1 --2- 172.21.15.159:0/3674937540 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fc02009b3a0 0x7fc02009d850 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7fc03000b5c0 tx=0x7fc030005fb0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:30.997 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:30.998+0000 7fc02b7fe700 1 -- 172.21.15.159:0/3674937540 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fc0240905b0 con 0x7fc034100790 2023-03-27T10:22:31.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:30 smithi161 ceph-mon[94333]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2023-03-27T10:22:31.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:30 smithi161 ceph-mon[94333]: osdmap e82: 12 total, 12 up, 12 in 2023-03-27T10:22:31.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:30 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/4279510122' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]: dispatch 2023-03-27T10:22:31.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:30 smithi159 ceph-mon[93039]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2023-03-27T10:22:31.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:30 smithi159 ceph-mon[93039]: osdmap e82: 12 total, 12 up, 12 in 2023-03-27T10:22:31.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:30 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/4279510122' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]: dispatch 2023-03-27T10:22:31.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:30 smithi171 ceph-mon[94808]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2023-03-27T10:22:31.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:30 smithi171 ceph-mon[94808]: osdmap e82: 12 total, 12 up, 12 in 2023-03-27T10:22:31.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:30 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/4279510122' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]: dispatch 2023-03-27T10:22:31.251 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.252+0000 7fc03c78f700 1 -- 172.21.15.159:0/3674937540 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7fc034061690 con 0x7fc034100790 2023-03-27T10:22:31.258 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.259+0000 7fc02b7fe700 1 -- 172.21.15.159:0/3674937540 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 2 and data pool 3 v14) v1 ==== 172+0+0 (secure 0 0 0) 0x7fc024020070 con 0x7fc034100790 2023-03-27T10:22:31.267 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.267+0000 7fc03c78f700 1 -- 172.21.15.159:0/3674937540 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fc02009b3a0 msgr2=0x7fc02009d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:31.268 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.267+0000 7fc03c78f700 1 --2- 172.21.15.159:0/3674937540 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fc02009b3a0 0x7fc02009d850 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7fc03000b5c0 tx=0x7fc030005fb0 comp rx=0 tx=0).stop 2023-03-27T10:22:31.268 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.267+0000 7fc03c78f700 1 -- 172.21.15.159:0/3674937540 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc034100790 msgr2=0x7fc034195af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:31.268 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.267+0000 7fc03c78f700 1 --2- 172.21.15.159:0/3674937540 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc034100790 0x7fc034195af0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7fc02400d950 tx=0x7fc02400dc60 comp rx=0 tx=0).stop 2023-03-27T10:22:31.268 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.267+0000 7fc03c78f700 1 -- 172.21.15.159:0/3674937540 shutdown_connections 2023-03-27T10:22:31.268 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.267+0000 7fc03c78f700 1 --2- 172.21.15.159:0/3674937540 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fc02009b3a0 0x7fc02009d850 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:31.268 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.267+0000 7fc03c78f700 1 --2- 172.21.15.159:0/3674937540 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fc0341059d0 0x7fc03419d0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:31.269 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.267+0000 7fc03c78f700 1 --2- 172.21.15.159:0/3674937540 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fc034100790 0x7fc034195af0 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:31.269 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.267+0000 7fc03c78f700 1 --2- 172.21.15.159:0/3674937540 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fc0341030b0 0x7fc034196030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:31.269 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.267+0000 7fc03c78f700 1 -- 172.21.15.159:0/3674937540 >> 172.21.15.159:0/3674937540 conn(0x7fc0340fa230 msgr2=0x7fc0341013d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:31.269 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.268+0000 7fc03c78f700 1 -- 172.21.15.159:0/3674937540 shutdown_connections 2023-03-27T10:22:31.269 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.268+0000 7fc03c78f700 1 -- 172.21.15.159:0/3674937540 wait complete. 2023-03-27T10:22:31.275 INFO:teuthology.orchestra.run.smithi159.stderr:new fs with metadata pool 2 and data pool 3 2023-03-27T10:22:31.292 DEBUG:tasks.cephfs.filesystem:EC profile is ['m=2', 'k=2', 'crush-failure-domain=osd'] 2023-03-27T10:22:31.292 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd erasure-code-profile set cephfs_data_ec m=2 k=2 crush-failure-domain=osd 2023-03-27T10:22:31.431 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.432+0000 7f65f17b7700 1 -- 172.21.15.159:0/1909418543 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f65ec0ff790 msgr2=0x7f65ec107040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:31.431 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.432+0000 7f65f17b7700 1 --2- 172.21.15.159:0/1909418543 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f65ec0ff790 0x7f65ec107040 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f65e400b0a0 tx=0x7f65e400b3b0 comp rx=0 tx=0).stop 2023-03-27T10:22:31.432 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.432+0000 7f65f17b7700 1 -- 172.21.15.159:0/1909418543 shutdown_connections 2023-03-27T10:22:31.432 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.432+0000 7f65f17b7700 1 --2- 172.21.15.159:0/1909418543 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f65ec0ff790 0x7f65ec107040 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:31.432 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.432+0000 7f65f17b7700 1 --2- 172.21.15.159:0/1909418543 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f65ec100190 0x7f65ec1005a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:31.432 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.432+0000 7f65f17b7700 1 --2- 172.21.15.159:0/1909418543 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f65ec0fede0 0x7f65ec0ff250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:31.432 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.432+0000 7f65f17b7700 1 -- 172.21.15.159:0/1909418543 >> 172.21.15.159:0/1909418543 conn(0x7f65ec0f99a0 msgr2=0x7f65ec0fbdb0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:31.432 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.432+0000 7f65f17b7700 1 -- 172.21.15.159:0/1909418543 shutdown_connections 2023-03-27T10:22:31.432 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.432+0000 7f65f17b7700 1 -- 172.21.15.159:0/1909418543 wait complete. 2023-03-27T10:22:31.433 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.432+0000 7f65f17b7700 1 Processor -- start 2023-03-27T10:22:31.434 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.432+0000 7f65f17b7700 1 -- start start 2023-03-27T10:22:31.434 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.432+0000 7f65f17b7700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f65ec0fede0 0x7f65ec102b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:31.434 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65f17b7700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f65ec0ff790 0x7f65ec1011d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:31.435 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65f17b7700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f65ec100190 0x7f65ec101710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:31.435 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65f17b7700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65ec101eb0 con 0x7f65ec0ff790 2023-03-27T10:22:31.435 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65f17b7700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65ec102020 con 0x7f65ec0fede0 2023-03-27T10:22:31.435 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65f17b7700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65ec19fc40 con 0x7f65ec100190 2023-03-27T10:22:31.435 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65ea7fc700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f65ec0ff790 0x7f65ec1011d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:31.435 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65ea7fc700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f65ec0ff790 0x7f65ec1011d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45642/0 (socket says 172.21.15.159:45642) 2023-03-27T10:22:31.435 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65ea7fc700 1 -- 172.21.15.159:0/874293059 learned_addr learned my addr 172.21.15.159:0/874293059 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:31.435 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65eb7fe700 1 --2- 172.21.15.159:0/874293059 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f65ec100190 0x7f65ec101710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:31.436 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65eaffd700 1 --2- 172.21.15.159:0/874293059 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f65ec0fede0 0x7f65ec102b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:31.436 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65ea7fc700 1 -- 172.21.15.159:0/874293059 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f65ec100190 msgr2=0x7f65ec101710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:31.436 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65ea7fc700 1 --2- 172.21.15.159:0/874293059 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f65ec100190 0x7f65ec101710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:31.436 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65ea7fc700 1 -- 172.21.15.159:0/874293059 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f65ec0fede0 msgr2=0x7f65ec102b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:31.436 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65ea7fc700 1 --2- 172.21.15.159:0/874293059 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f65ec0fede0 0x7f65ec102b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:31.436 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65ea7fc700 1 -- 172.21.15.159:0/874293059 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f65e4009d00 con 0x7f65ec0ff790 2023-03-27T10:22:31.436 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65eb7fe700 1 --2- 172.21.15.159:0/874293059 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f65ec100190 0x7f65ec101710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-03-27T10:22:31.437 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65ea7fc700 1 --2- 172.21.15.159:0/874293059 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f65ec0ff790 0x7f65ec1011d0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f65dc00e9c0 tx=0x7f65dc00ecd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:31.437 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65d3fff700 1 -- 172.21.15.159:0/874293059 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65dc0097b0 con 0x7f65ec0ff790 2023-03-27T10:22:31.437 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65f17b7700 1 -- 172.21.15.159:0/874293059 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f65ec10a550 con 0x7f65ec0ff790 2023-03-27T10:22:31.437 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65d3fff700 1 -- 172.21.15.159:0/874293059 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f65dc00ee60 con 0x7f65ec0ff790 2023-03-27T10:22:31.437 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65d3fff700 1 -- 172.21.15.159:0/874293059 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65dc0054f0 con 0x7f65ec0ff790 2023-03-27T10:22:31.437 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.433+0000 7f65f17b7700 1 -- 172.21.15.159:0/874293059 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f65ec10a3c0 con 0x7f65ec0ff790 2023-03-27T10:22:31.437 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.434+0000 7f65f17b7700 1 -- 172.21.15.159:0/874293059 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f65ec1a02b0 con 0x7f65ec0ff790 2023-03-27T10:22:31.438 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.435+0000 7f65d3fff700 1 -- 172.21.15.159:0/874293059 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f65dc01e020 con 0x7f65ec0ff790 2023-03-27T10:22:31.438 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.436+0000 7f65d3fff700 1 --2- 172.21.15.159:0/874293059 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f65d409b3a0 0x7f65d409d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:31.438 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.436+0000 7f65eaffd700 1 --2- 172.21.15.159:0/874293059 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f65d409b3a0 0x7f65d409d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:31.438 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.436+0000 7f65d3fff700 1 -- 172.21.15.159:0/874293059 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f65dc014070 con 0x7f65ec0ff790 2023-03-27T10:22:31.439 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.436+0000 7f65eaffd700 1 --2- 172.21.15.159:0/874293059 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f65d409b3a0 0x7f65d409d850 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f65e0005950 tx=0x7f65e000b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:31.439 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.440+0000 7f65d3fff700 1 -- 172.21.15.159:0/874293059 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f65dc090a50 con 0x7f65ec0ff790 2023-03-27T10:22:31.646 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:31.647+0000 7f65f17b7700 1 -- 172.21.15.159:0/874293059 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]} v 0) v1 -- 0x7f65ec1a0950 con 0x7f65ec0ff790 2023-03-27T10:22:32.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:31 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/4279510122' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]': finished 2023-03-27T10:22:32.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:31 smithi161 ceph-mon[94333]: osdmap e83: 12 total, 12 up, 12 in 2023-03-27T10:22:32.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:31 smithi161 ceph-mon[94333]: pgmap v255: 97 pgs: 25 creating+peering, 3 active+clean, 69 unknown; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:32.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:31 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3674937540' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2023-03-27T10:22:32.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:31 smithi161 ceph-mon[94333]: daemon mds.b assigned to filesystem cephfs as rank 0 2023-03-27T10:22:32.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:31 smithi161 ceph-mon[94333]: osdmap e84: 12 total, 12 up, 12 in 2023-03-27T10:22:32.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:31 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3674937540' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2023-03-27T10:22:32.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:31 smithi161 ceph-mon[94333]: fsmap cephfs:1 {0=b=up:creating} 11 up:standby 2023-03-27T10:22:32.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:31 smithi161 ceph-mon[94333]: daemon mds.b is now active in filesystem cephfs as rank 0 2023-03-27T10:22:32.085 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:31 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/874293059' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]: dispatch 2023-03-27T10:22:32.114 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:31 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/4279510122' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]': finished 2023-03-27T10:22:32.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:31 smithi159 ceph-mon[93039]: osdmap e83: 12 total, 12 up, 12 in 2023-03-27T10:22:32.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:31 smithi159 ceph-mon[93039]: pgmap v255: 97 pgs: 25 creating+peering, 3 active+clean, 69 unknown; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:32.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:31 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3674937540' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2023-03-27T10:22:32.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:31 smithi159 ceph-mon[93039]: daemon mds.b assigned to filesystem cephfs as rank 0 2023-03-27T10:22:32.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:31 smithi159 ceph-mon[93039]: osdmap e84: 12 total, 12 up, 12 in 2023-03-27T10:22:32.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:31 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3674937540' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2023-03-27T10:22:32.115 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:31 smithi159 ceph-mon[93039]: fsmap cephfs:1 {0=b=up:creating} 11 up:standby 2023-03-27T10:22:32.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:31 smithi159 ceph-mon[93039]: daemon mds.b is now active in filesystem cephfs as rank 0 2023-03-27T10:22:32.116 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:31 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/874293059' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]: dispatch 2023-03-27T10:22:32.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:31 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/4279510122' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]': finished 2023-03-27T10:22:32.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:31 smithi171 ceph-mon[94808]: osdmap e83: 12 total, 12 up, 12 in 2023-03-27T10:22:32.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:31 smithi171 ceph-mon[94808]: pgmap v255: 97 pgs: 25 creating+peering, 3 active+clean, 69 unknown; 577 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail 2023-03-27T10:22:32.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:31 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3674937540' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2023-03-27T10:22:32.226 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:31 smithi171 ceph-mon[94808]: daemon mds.b assigned to filesystem cephfs as rank 0 2023-03-27T10:22:32.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:31 smithi171 ceph-mon[94808]: osdmap e84: 12 total, 12 up, 12 in 2023-03-27T10:22:32.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:31 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3674937540' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2023-03-27T10:22:32.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:31 smithi171 ceph-mon[94808]: fsmap cephfs:1 {0=b=up:creating} 11 up:standby 2023-03-27T10:22:32.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:31 smithi171 ceph-mon[94808]: daemon mds.b is now active in filesystem cephfs as rank 0 2023-03-27T10:22:32.227 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:31 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/874293059' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]: dispatch 2023-03-27T10:22:32.266 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.267+0000 7f65d3fff700 1 -- 172.21.15.159:0/874293059 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]=0 v85) v1 ==== 157+0+0 (secure 0 0 0) 0x7f65dc094070 con 0x7f65ec0ff790 2023-03-27T10:22:32.269 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.269+0000 7f65f17b7700 1 -- 172.21.15.159:0/874293059 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f65d409b3a0 msgr2=0x7f65d409d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:32.269 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.269+0000 7f65f17b7700 1 --2- 172.21.15.159:0/874293059 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f65d409b3a0 0x7f65d409d850 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f65e0005950 tx=0x7f65e000b410 comp rx=0 tx=0).stop 2023-03-27T10:22:32.269 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.269+0000 7f65f17b7700 1 -- 172.21.15.159:0/874293059 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f65ec0ff790 msgr2=0x7f65ec1011d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:32.269 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.269+0000 7f65f17b7700 1 --2- 172.21.15.159:0/874293059 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f65ec0ff790 0x7f65ec1011d0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f65dc00e9c0 tx=0x7f65dc00ecd0 comp rx=0 tx=0).stop 2023-03-27T10:22:32.269 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.269+0000 7f65f17b7700 1 -- 172.21.15.159:0/874293059 shutdown_connections 2023-03-27T10:22:32.269 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.269+0000 7f65f17b7700 1 --2- 172.21.15.159:0/874293059 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f65d409b3a0 0x7f65d409d850 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:32.270 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.269+0000 7f65f17b7700 1 --2- 172.21.15.159:0/874293059 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f65ec100190 0x7f65ec101710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:32.270 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.269+0000 7f65f17b7700 1 --2- 172.21.15.159:0/874293059 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f65ec0fede0 0x7f65ec102b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:32.270 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.269+0000 7f65f17b7700 1 --2- 172.21.15.159:0/874293059 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f65ec0ff790 0x7f65ec1011d0 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:32.270 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.269+0000 7f65f17b7700 1 -- 172.21.15.159:0/874293059 >> 172.21.15.159:0/874293059 conn(0x7f65ec0f99a0 msgr2=0x7f65ec0fdcb0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:32.270 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.269+0000 7f65f17b7700 1 -- 172.21.15.159:0/874293059 shutdown_connections 2023-03-27T10:22:32.270 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.269+0000 7f65f17b7700 1 -- 172.21.15.159:0/874293059 wait complete. 2023-03-27T10:22:32.283 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data_ec erasure cephfs_data_ec --pg_num_min 64 --target_size_ratio 0.9 2023-03-27T10:22:32.418 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.418+0000 7fa304236700 1 -- 172.21.15.159:0/2858912524 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa2fc1019e0 msgr2=0x7fa2fc1082a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:32.419 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.418+0000 7fa304236700 1 --2- 172.21.15.159:0/2858912524 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa2fc1019e0 0x7fa2fc1082a0 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7fa2f400b0a0 tx=0x7fa2f400b3b0 comp rx=0 tx=0).stop 2023-03-27T10:22:32.419 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.418+0000 7fa304236700 1 -- 172.21.15.159:0/2858912524 shutdown_connections 2023-03-27T10:22:32.419 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.419+0000 7fa304236700 1 --2- 172.21.15.159:0/2858912524 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa2fc0fe080 0x7fa2fc0fe490 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:32.419 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.419+0000 7fa304236700 1 --2- 172.21.15.159:0/2858912524 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa2fc1019e0 0x7fa2fc1082a0 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:32.419 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.419+0000 7fa304236700 1 --2- 172.21.15.159:0/2858912524 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa2fc101050 0x7fa2fc1014a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:32.419 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.419+0000 7fa304236700 1 -- 172.21.15.159:0/2858912524 >> 172.21.15.159:0/2858912524 conn(0x7fa2fc0f9a80 msgr2=0x7fa2fc0fbe90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:32.419 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.419+0000 7fa304236700 1 -- 172.21.15.159:0/2858912524 shutdown_connections 2023-03-27T10:22:32.420 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.419+0000 7fa304236700 1 -- 172.21.15.159:0/2858912524 wait complete. 2023-03-27T10:22:32.420 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.419+0000 7fa304236700 1 Processor -- start 2023-03-27T10:22:32.420 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.419+0000 7fa304236700 1 -- start start 2023-03-27T10:22:32.421 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.419+0000 7fa304236700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa2fc0fe080 0x7fa2fc195b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:32.422 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.419+0000 7fa304236700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa2fc101050 0x7fa2fc196080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:32.422 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.419+0000 7fa304236700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa2fc1019e0 0x7fa2fc19d0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:32.422 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.419+0000 7fa304236700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa2fc19d6c0 con 0x7fa2fc0fe080 2023-03-27T10:22:32.422 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.419+0000 7fa304236700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa2fc19d800 con 0x7fa2fc1019e0 2023-03-27T10:22:32.422 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.419+0000 7fa304236700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa2fc19d990 con 0x7fa2fc101050 2023-03-27T10:22:32.422 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.419+0000 7fa301fd2700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa2fc0fe080 0x7fa2fc195b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:32.422 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.420+0000 7fa301fd2700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa2fc0fe080 0x7fa2fc195b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45652/0 (socket says 172.21.15.159:45652) 2023-03-27T10:22:32.422 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.420+0000 7fa301fd2700 1 -- 172.21.15.159:0/3773497711 learned_addr learned my addr 172.21.15.159:0/3773497711 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:32.423 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.420+0000 7fa3017d1700 1 --2- 172.21.15.159:0/3773497711 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa2fc101050 0x7fa2fc196080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:32.423 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.420+0000 7fa3027d3700 1 --2- 172.21.15.159:0/3773497711 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa2fc1019e0 0x7fa2fc19d0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:32.423 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.420+0000 7fa301fd2700 1 -- 172.21.15.159:0/3773497711 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa2fc101050 msgr2=0x7fa2fc196080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:32.423 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.420+0000 7fa301fd2700 1 --2- 172.21.15.159:0/3773497711 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa2fc101050 0x7fa2fc196080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:32.423 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.420+0000 7fa301fd2700 1 -- 172.21.15.159:0/3773497711 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa2fc1019e0 msgr2=0x7fa2fc19d0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:32.423 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.420+0000 7fa301fd2700 1 --2- 172.21.15.159:0/3773497711 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa2fc1019e0 0x7fa2fc19d0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:32.423 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.420+0000 7fa301fd2700 1 -- 172.21.15.159:0/3773497711 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa2f4009d00 con 0x7fa2fc0fe080 2023-03-27T10:22:32.424 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.420+0000 7fa3027d3700 1 --2- 172.21.15.159:0/3773497711 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa2fc1019e0 0x7fa2fc19d0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-03-27T10:22:32.424 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.420+0000 7fa301fd2700 1 --2- 172.21.15.159:0/3773497711 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa2fc0fe080 0x7fa2fc195b40 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7fa2f800e970 tx=0x7fa2f800ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:32.424 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.420+0000 7fa2f2ffd700 1 -- 172.21.15.159:0/3773497711 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa2f8009790 con 0x7fa2fc0fe080 2023-03-27T10:22:32.424 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.420+0000 7fa2f2ffd700 1 -- 172.21.15.159:0/3773497711 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa2f800eec0 con 0x7fa2fc0fe080 2023-03-27T10:22:32.424 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.420+0000 7fa304236700 1 -- 172.21.15.159:0/3773497711 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa2fc10b730 con 0x7fa2fc0fe080 2023-03-27T10:22:32.424 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.420+0000 7fa2f2ffd700 1 -- 172.21.15.159:0/3773497711 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa2f8003cf0 con 0x7fa2fc0fe080 2023-03-27T10:22:32.424 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.420+0000 7fa304236700 1 -- 172.21.15.159:0/3773497711 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa2fc10b5a0 con 0x7fa2fc0fe080 2023-03-27T10:22:32.424 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.421+0000 7fa304236700 1 -- 172.21.15.159:0/3773497711 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa2fc106d10 con 0x7fa2fc0fe080 2023-03-27T10:22:32.425 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.422+0000 7fa2f2ffd700 1 -- 172.21.15.159:0/3773497711 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7fa2f801e020 con 0x7fa2fc0fe080 2023-03-27T10:22:32.425 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.423+0000 7fa2f2ffd700 1 --2- 172.21.15.159:0/3773497711 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa2e809b3a0 0x7fa2e809d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:32.425 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.423+0000 7fa2f2ffd700 1 -- 172.21.15.159:0/3773497711 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(85..85 src has 1..85) v4 ==== 8205+0+0 (secure 0 0 0) 0x7fa2f8014070 con 0x7fa2fc0fe080 2023-03-27T10:22:32.426 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.423+0000 7fa3017d1700 1 --2- 172.21.15.159:0/3773497711 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa2e809b3a0 0x7fa2e809d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:32.426 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.423+0000 7fa3017d1700 1 --2- 172.21.15.159:0/3773497711 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa2e809b3a0 0x7fa2e809d850 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7fa2ec009c60 tx=0x7fa2ec009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:32.427 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.427+0000 7fa2f2ffd700 1 -- 172.21.15.159:0/3773497711 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fa2f8090c30 con 0x7fa2fc0fe080 2023-03-27T10:22:32.616 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:32.617+0000 7fa304236700 1 -- 172.21.15.159:0/3773497711 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7fa2fc061690 con 0x7fa2fc0fe080 2023-03-27T10:22:33.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:33 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/874293059' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]': finished 2023-03-27T10:22:33.584 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:33 smithi161 ceph-mon[94333]: osdmap e85: 12 total, 12 up, 12 in 2023-03-27T10:22:33.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:33 smithi161 ceph-mon[94333]: mds.? [v2:172.21.15.161:6838/2491878460,v1:172.21.15.161:6839/2491878460] up:active 2023-03-27T10:22:33.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:33 smithi161 ceph-mon[94333]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-03-27T10:22:33.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:33 smithi161 ceph-mon[94333]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-03-27T10:22:33.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.585 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.586 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.586 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:33.586 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:22:33.586 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:33 smithi161 ceph-mon[94333]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.586 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:33 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3773497711' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2023-03-27T10:22:33.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:33 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/874293059' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]': finished 2023-03-27T10:22:33.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:33 smithi159 ceph-mon[93039]: osdmap e85: 12 total, 12 up, 12 in 2023-03-27T10:22:33.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:33 smithi159 ceph-mon[93039]: mds.? [v2:172.21.15.161:6838/2491878460,v1:172.21.15.161:6839/2491878460] up:active 2023-03-27T10:22:33.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:33 smithi159 ceph-mon[93039]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-03-27T10:22:33.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:33 smithi159 ceph-mon[93039]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-03-27T10:22:33.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:33.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:22:33.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:33 smithi159 ceph-mon[93039]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.616 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:33 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3773497711' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2023-03-27T10:22:33.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:33 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/874293059' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]': finished 2023-03-27T10:22:33.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:33 smithi171 ceph-mon[94808]: osdmap e85: 12 total, 12 up, 12 in 2023-03-27T10:22:33.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:33 smithi171 ceph-mon[94808]: mds.? [v2:172.21.15.161:6838/2491878460,v1:172.21.15.161:6839/2491878460] up:active 2023-03-27T10:22:33.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:33 smithi171 ceph-mon[94808]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-03-27T10:22:33.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:33 smithi171 ceph-mon[94808]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-03-27T10:22:33.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-03-27T10:22:33.728 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-03-27T10:22:33.728 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:33 smithi171 ceph-mon[94808]: from='mgr.14152 172.21.15.159:0/2383151484' entity='mgr.x' 2023-03-27T10:22:33.728 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:33 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3773497711' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2023-03-27T10:22:34.423 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.423+0000 7fa2f2ffd700 1 -- 172.21.15.159:0/3773497711 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data_ec' created v87) v1 ==== 228+0+0 (secure 0 0 0) 0x7fa2f8094250 con 0x7fa2fc0fe080 2023-03-27T10:22:34.427 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.427+0000 7fa304236700 1 -- 172.21.15.159:0/3773497711 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa2e809b3a0 msgr2=0x7fa2e809d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:34.427 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.428+0000 7fa304236700 1 --2- 172.21.15.159:0/3773497711 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa2e809b3a0 0x7fa2e809d850 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7fa2ec009c60 tx=0x7fa2ec009260 comp rx=0 tx=0).stop 2023-03-27T10:22:34.428 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.428+0000 7fa304236700 1 -- 172.21.15.159:0/3773497711 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa2fc0fe080 msgr2=0x7fa2fc195b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:34.428 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.428+0000 7fa304236700 1 --2- 172.21.15.159:0/3773497711 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa2fc0fe080 0x7fa2fc195b40 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7fa2f800e970 tx=0x7fa2f800ec80 comp rx=0 tx=0).stop 2023-03-27T10:22:34.428 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.428+0000 7fa304236700 1 -- 172.21.15.159:0/3773497711 shutdown_connections 2023-03-27T10:22:34.428 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.428+0000 7fa304236700 1 --2- 172.21.15.159:0/3773497711 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa2e809b3a0 0x7fa2e809d850 secure :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7fa2ec009c60 tx=0x7fa2ec009260 comp rx=0 tx=0).stop 2023-03-27T10:22:34.428 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.428+0000 7fa304236700 1 --2- 172.21.15.159:0/3773497711 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa2fc1019e0 0x7fa2fc19d0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:34.428 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.428+0000 7fa304236700 1 --2- 172.21.15.159:0/3773497711 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa2fc0fe080 0x7fa2fc195b40 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:34.428 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.428+0000 7fa304236700 1 --2- 172.21.15.159:0/3773497711 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa2fc101050 0x7fa2fc196080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:34.429 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.428+0000 7fa304236700 1 -- 172.21.15.159:0/3773497711 >> 172.21.15.159:0/3773497711 conn(0x7fa2fc0f9a80 msgr2=0x7fa2fc0fbc40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:34.429 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.429+0000 7fa304236700 1 -- 172.21.15.159:0/3773497711 shutdown_connections 2023-03-27T10:22:34.429 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.429+0000 7fa304236700 1 -- 172.21.15.159:0/3773497711 wait complete. 2023-03-27T10:22:34.431 INFO:teuthology.orchestra.run.smithi159.stderr:pool 'cephfs_data_ec' created 2023-03-27T10:22:34.446 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool set cephfs_data_ec allow_ec_overwrites true 2023-03-27T10:22:34.576 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.577+0000 7fa3e38fc700 1 -- 172.21.15.159:0/2542998022 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa3dc100820 msgr2=0x7fa3dc102c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:34.576 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.577+0000 7fa3e38fc700 1 --2- 172.21.15.159:0/2542998022 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa3dc100820 0x7fa3dc102c00 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7fa3d8009990 tx=0x7fa3d8009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:34.577 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.577+0000 7fa3e38fc700 1 -- 172.21.15.159:0/2542998022 shutdown_connections 2023-03-27T10:22:34.577 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.577+0000 7fa3e38fc700 1 --2- 172.21.15.159:0/2542998022 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa3dc105a60 0x7fa3dc107e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:34.577 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.577+0000 7fa3e38fc700 1 --2- 172.21.15.159:0/2542998022 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa3dc100820 0x7fa3dc102c00 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:34.577 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.577+0000 7fa3e38fc700 1 --2- 172.21.15.159:0/2542998022 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa3dc103140 0x7fa3dc105520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:34.577 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.577+0000 7fa3e38fc700 1 -- 172.21.15.159:0/2542998022 >> 172.21.15.159:0/2542998022 conn(0x7fa3dc0fa220 msgr2=0x7fa3dc0fc670 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:34.577 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.577+0000 7fa3e38fc700 1 -- 172.21.15.159:0/2542998022 shutdown_connections 2023-03-27T10:22:34.577 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.577+0000 7fa3e38fc700 1 -- 172.21.15.159:0/2542998022 wait complete. 2023-03-27T10:22:34.579 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.577+0000 7fa3e38fc700 1 Processor -- start 2023-03-27T10:22:34.579 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.577+0000 7fa3e38fc700 1 -- start start 2023-03-27T10:22:34.579 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.577+0000 7fa3e38fc700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa3dc100820 0x7fa3dc195a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:34.580 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.577+0000 7fa3e38fc700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa3dc103140 0x7fa3dc195f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:34.580 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.577+0000 7fa3e38fc700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa3dc105a60 0x7fa3dc19cfd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:34.580 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.577+0000 7fa3e38fc700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa3dc19d5a0 con 0x7fa3dc100820 2023-03-27T10:22:34.580 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3e38fc700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa3dc19d6e0 con 0x7fa3dc103140 2023-03-27T10:22:34.580 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3e38fc700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa3dc19d870 con 0x7fa3dc105a60 2023-03-27T10:22:34.580 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3e1698700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa3dc100820 0x7fa3dc195a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:34.580 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3e1698700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa3dc100820 0x7fa3dc195a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45668/0 (socket says 172.21.15.159:45668) 2023-03-27T10:22:34.580 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3e1698700 1 -- 172.21.15.159:0/999449511 learned_addr learned my addr 172.21.15.159:0/999449511 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:34.581 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3e0e97700 1 --2- 172.21.15.159:0/999449511 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa3dc103140 0x7fa3dc195f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:34.581 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3e1e99700 1 --2- 172.21.15.159:0/999449511 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa3dc105a60 0x7fa3dc19cfd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:34.581 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3e1698700 1 -- 172.21.15.159:0/999449511 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa3dc105a60 msgr2=0x7fa3dc19cfd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:34.581 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3e1698700 1 --2- 172.21.15.159:0/999449511 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa3dc105a60 0x7fa3dc19cfd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:34.581 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3e1698700 1 -- 172.21.15.159:0/999449511 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa3dc103140 msgr2=0x7fa3dc195f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:34.581 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3e1698700 1 --2- 172.21.15.159:0/999449511 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa3dc103140 0x7fa3dc195f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:34.581 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3e1698700 1 -- 172.21.15.159:0/999449511 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa3d80095f0 con 0x7fa3dc100820 2023-03-27T10:22:34.582 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3e1698700 1 --2- 172.21.15.159:0/999449511 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa3dc100820 0x7fa3dc195a20 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7fa3d80059d0 tx=0x7fa3d8004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:34.582 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3d27fc700 1 -- 172.21.15.159:0/999449511 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa3d801d070 con 0x7fa3dc100820 2023-03-27T10:22:34.582 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3d27fc700 1 -- 172.21.15.159:0/999449511 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa3d800fb90 con 0x7fa3dc100820 2023-03-27T10:22:34.582 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3e38fc700 1 -- 172.21.15.159:0/999449511 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa3dc10b660 con 0x7fa3dc100820 2023-03-27T10:22:34.582 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3d27fc700 1 -- 172.21.15.159:0/999449511 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa3d80173f0 con 0x7fa3dc100820 2023-03-27T10:22:34.582 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.578+0000 7fa3e38fc700 1 -- 172.21.15.159:0/999449511 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa3dc10b4d0 con 0x7fa3dc100820 2023-03-27T10:22:34.582 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.579+0000 7fa3e38fc700 1 -- 172.21.15.159:0/999449511 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa3dc0fbd90 con 0x7fa3dc100820 2023-03-27T10:22:34.583 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.580+0000 7fa3d27fc700 1 -- 172.21.15.159:0/999449511 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7fa3d8029030 con 0x7fa3dc100820 2023-03-27T10:22:34.583 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.581+0000 7fa3d27fc700 1 --2- 172.21.15.159:0/999449511 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa3c809b3a0 0x7fa3c809d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:34.583 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.581+0000 7fa3e0e97700 1 --2- 172.21.15.159:0/999449511 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa3c809b3a0 0x7fa3c809d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:34.583 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.581+0000 7fa3d27fc700 1 -- 172.21.15.159:0/999449511 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(87..87 src has 1..87) v4 ==== 8698+0+0 (secure 0 0 0) 0x7fa3d80c65a0 con 0x7fa3dc100820 2023-03-27T10:22:34.584 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.581+0000 7fa3e0e97700 1 --2- 172.21.15.159:0/999449511 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa3c809b3a0 0x7fa3c809d850 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7fa3cc009c60 tx=0x7fa3cc009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:34.584 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.584+0000 7fa3d27fc700 1 -- 172.21.15.159:0/999449511 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fa3d8091d80 con 0x7fa3dc100820 2023-03-27T10:22:34.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:34 smithi171 ceph-mon[94808]: pgmap v258: 97 pgs: 25 creating+peering, 24 active+clean, 48 unknown; 579 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail; 1023 B/s wr, 3 op/s 2023-03-27T10:22:34.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:34 smithi171 ceph-mon[94808]: osdmap e86: 12 total, 12 up, 12 in 2023-03-27T10:22:34.766 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:34.767+0000 7fa3e38fc700 1 -- 172.21.15.159:0/999449511 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"} v 0) v1 -- 0x7fa3dc061610 con 0x7fa3dc100820 2023-03-27T10:22:34.833 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:34 smithi161 ceph-mon[94333]: pgmap v258: 97 pgs: 25 creating+peering, 24 active+clean, 48 unknown; 579 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail; 1023 B/s wr, 3 op/s 2023-03-27T10:22:34.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:34 smithi161 ceph-mon[94333]: osdmap e86: 12 total, 12 up, 12 in 2023-03-27T10:22:34.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:34 smithi159 ceph-mon[93039]: pgmap v258: 97 pgs: 25 creating+peering, 24 active+clean, 48 unknown; 579 KiB data, 74 MiB used, 1.0 TiB / 1.0 TiB avail; 1023 B/s wr, 3 op/s 2023-03-27T10:22:34.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:34 smithi159 ceph-mon[93039]: osdmap e86: 12 total, 12 up, 12 in 2023-03-27T10:22:35.429 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.430+0000 7fa3d27fc700 1 -- 172.21.15.159:0/999449511 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]=0 set pool 4 allow_ec_overwrites to true v88) v1 ==== 169+0+0 (secure 0 0 0) 0x7fa3d80953a0 con 0x7fa3dc100820 2023-03-27T10:22:35.434 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.434+0000 7fa3e38fc700 1 -- 172.21.15.159:0/999449511 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa3c809b3a0 msgr2=0x7fa3c809d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:35.434 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.434+0000 7fa3e38fc700 1 --2- 172.21.15.159:0/999449511 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa3c809b3a0 0x7fa3c809d850 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7fa3cc009c60 tx=0x7fa3cc009260 comp rx=0 tx=0).stop 2023-03-27T10:22:35.434 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.434+0000 7fa3e38fc700 1 -- 172.21.15.159:0/999449511 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa3dc100820 msgr2=0x7fa3dc195a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:35.434 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.434+0000 7fa3e38fc700 1 --2- 172.21.15.159:0/999449511 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa3dc100820 0x7fa3dc195a20 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7fa3d80059d0 tx=0x7fa3d8004730 comp rx=0 tx=0).stop 2023-03-27T10:22:35.442 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.443+0000 7fa3e38fc700 1 -- 172.21.15.159:0/999449511 shutdown_connections 2023-03-27T10:22:35.442 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.443+0000 7fa3e38fc700 1 --2- 172.21.15.159:0/999449511 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa3c809b3a0 0x7fa3c809d850 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.442 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.443+0000 7fa3e38fc700 1 --2- 172.21.15.159:0/999449511 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa3dc105a60 0x7fa3dc19cfd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.442 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.443+0000 7fa3e38fc700 1 --2- 172.21.15.159:0/999449511 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa3dc100820 0x7fa3dc195a20 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.442 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.443+0000 7fa3e38fc700 1 --2- 172.21.15.159:0/999449511 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa3dc103140 0x7fa3dc195f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.443 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.443+0000 7fa3e38fc700 1 -- 172.21.15.159:0/999449511 >> 172.21.15.159:0/999449511 conn(0x7fa3dc0fa220 msgr2=0x7fa3dc101460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:35.444 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.445+0000 7fa3e38fc700 1 -- 172.21.15.159:0/999449511 shutdown_connections 2023-03-27T10:22:35.445 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.446+0000 7fa3e38fc700 1 -- 172.21.15.159:0/999449511 wait complete. 2023-03-27T10:22:35.446 INFO:teuthology.orchestra.run.smithi159.stderr:set pool 4 allow_ec_overwrites to true 2023-03-27T10:22:35.461 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs add_data_pool cephfs cephfs_data_ec 2023-03-27T10:22:35.588 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.588+0000 7f7958304700 1 -- 172.21.15.159:0/2229898996 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7950100810 msgr2=0x7f79501070b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:35.588 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.588+0000 7f7958304700 1 --2- 172.21.15.159:0/2229898996 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7950100810 0x7f79501070b0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f794800b0a0 tx=0x7f794800b3b0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.588 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.588+0000 7f7958304700 1 -- 172.21.15.159:0/2229898996 shutdown_connections 2023-03-27T10:22:35.588 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.588+0000 7f7958304700 1 --2- 172.21.15.159:0/2229898996 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f79500febd0 0x7f79500fefe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.589 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.588+0000 7f7958304700 1 --2- 172.21.15.159:0/2229898996 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f7950100810 0x7f79501070b0 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.589 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.588+0000 7f7958304700 1 --2- 172.21.15.159:0/2229898996 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f79500ffe60 0x7f79501002d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.589 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.588+0000 7f7958304700 1 -- 172.21.15.159:0/2229898996 >> 172.21.15.159:0/2229898996 conn(0x7f79500fa210 msgr2=0x7f79500fc620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:35.589 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.588+0000 7f7958304700 1 -- 172.21.15.159:0/2229898996 shutdown_connections 2023-03-27T10:22:35.589 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.588+0000 7f7958304700 1 -- 172.21.15.159:0/2229898996 wait complete. 2023-03-27T10:22:35.591 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.589+0000 7f7958304700 1 Processor -- start 2023-03-27T10:22:35.591 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.589+0000 7f7958304700 1 -- start start 2023-03-27T10:22:35.591 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.589+0000 7f7958304700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f79500febd0 0x7f7950068450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:35.591 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.589+0000 7f7958304700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f79500ffe60 0x7f795019a870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:35.592 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.589+0000 7f7958304700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7950100810 0x7f795019cec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:35.592 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.589+0000 7f7958304700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f795019d690 con 0x7f79500febd0 2023-03-27T10:22:35.592 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.589+0000 7f7958304700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f795019d800 con 0x7f79500ffe60 2023-03-27T10:22:35.592 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.589+0000 7f7958304700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f795019d9c0 con 0x7f7950100810 2023-03-27T10:22:35.592 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.589+0000 7f79560a0700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f79500febd0 0x7f7950068450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:35.592 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.589+0000 7f79560a0700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f79500febd0 0x7f7950068450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45690/0 (socket says 172.21.15.159:45690) 2023-03-27T10:22:35.592 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.589+0000 7f79560a0700 1 -- 172.21.15.159:0/2245674882 learned_addr learned my addr 172.21.15.159:0/2245674882 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:35.592 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.589+0000 7f795589f700 1 --2- 172.21.15.159:0/2245674882 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f79500ffe60 0x7f795019a870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:35.593 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.589+0000 7f79568a1700 1 --2- 172.21.15.159:0/2245674882 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7950100810 0x7f795019cec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:35.593 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.590+0000 7f79560a0700 1 -- 172.21.15.159:0/2245674882 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7950100810 msgr2=0x7f795019cec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:35.593 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.590+0000 7f79560a0700 1 --2- 172.21.15.159:0/2245674882 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7950100810 0x7f795019cec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.593 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.590+0000 7f79560a0700 1 -- 172.21.15.159:0/2245674882 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f79500ffe60 msgr2=0x7f795019a870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:35.593 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.590+0000 7f79560a0700 1 --2- 172.21.15.159:0/2245674882 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f79500ffe60 0x7f795019a870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.593 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.590+0000 7f79560a0700 1 -- 172.21.15.159:0/2245674882 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7948009d00 con 0x7f79500febd0 2023-03-27T10:22:35.593 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.590+0000 7f79560a0700 1 --2- 172.21.15.159:0/2245674882 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f79500febd0 0x7f7950068450 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f794c00d950 tx=0x7f794c00dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:35.594 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.590+0000 7f79477fe700 1 -- 172.21.15.159:0/2245674882 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f794c009940 con 0x7f79500febd0 2023-03-27T10:22:35.594 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.590+0000 7f79477fe700 1 -- 172.21.15.159:0/2245674882 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f794c010460 con 0x7f79500febd0 2023-03-27T10:22:35.594 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.590+0000 7f7958304700 1 -- 172.21.15.159:0/2245674882 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f795010a530 con 0x7f79500febd0 2023-03-27T10:22:35.594 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.590+0000 7f79477fe700 1 -- 172.21.15.159:0/2245674882 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f794c00f620 con 0x7f79500febd0 2023-03-27T10:22:35.594 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.590+0000 7f7958304700 1 -- 172.21.15.159:0/2245674882 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f795010a3a0 con 0x7f79500febd0 2023-03-27T10:22:35.594 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.591+0000 7f7958304700 1 -- 172.21.15.159:0/2245674882 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7950105bf0 con 0x7f79500febd0 2023-03-27T10:22:35.594 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.592+0000 7f79477fe700 1 -- 172.21.15.159:0/2245674882 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f794c022070 con 0x7f79500febd0 2023-03-27T10:22:35.595 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.592+0000 7f79477fe700 1 --2- 172.21.15.159:0/2245674882 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f793c09b3a0 0x7f793c09d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:35.595 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.592+0000 7f795589f700 1 --2- 172.21.15.159:0/2245674882 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f793c09b3a0 0x7f793c09d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:35.595 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.592+0000 7f79477fe700 1 -- 172.21.15.159:0/2245674882 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8698+0+0 (secure 0 0 0) 0x7f794c0c4e90 con 0x7f79500febd0 2023-03-27T10:22:35.595 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.593+0000 7f795589f700 1 --2- 172.21.15.159:0/2245674882 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f793c09b3a0 0x7f793c09d850 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f79500681c0 tx=0x7f794000b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:35.596 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.596+0000 7f79477fe700 1 -- 172.21.15.159:0/2245674882 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f794c0905e0 con 0x7f79500febd0 2023-03-27T10:22:35.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:35 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3773497711' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2023-03-27T10:22:35.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:35 smithi171 ceph-mon[94808]: osdmap e87: 12 total, 12 up, 12 in 2023-03-27T10:22:35.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:35 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/999449511' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]: dispatch 2023-03-27T10:22:35.826 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.827+0000 7f7958304700 1 -- 172.21.15.159:0/2245674882 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"} v 0) v1 -- 0x7f795002cdb0 con 0x7f79500febd0 2023-03-27T10:22:35.831 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.832+0000 7f79477fe700 1 -- 172.21.15.159:0/2245674882 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]=0 added data pool 4 to fsmap v17) v1 ==== 137+0+0 (secure 0 0 0) 0x7f794c020020 con 0x7f79500febd0 2023-03-27T10:22:35.833 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.834+0000 7f7958304700 1 -- 172.21.15.159:0/2245674882 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f793c09b3a0 msgr2=0x7f793c09d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:35.834 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.834+0000 7f7958304700 1 --2- 172.21.15.159:0/2245674882 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f793c09b3a0 0x7f793c09d850 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f79500681c0 tx=0x7f794000b410 comp rx=0 tx=0).stop 2023-03-27T10:22:35.834 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.834+0000 7f7958304700 1 -- 172.21.15.159:0/2245674882 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f79500febd0 msgr2=0x7f7950068450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:35.834 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.834+0000 7f7958304700 1 --2- 172.21.15.159:0/2245674882 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f79500febd0 0x7f7950068450 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f794c00d950 tx=0x7f794c00dc60 comp rx=0 tx=0).stop 2023-03-27T10:22:35.834 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.834+0000 7f7958304700 1 -- 172.21.15.159:0/2245674882 shutdown_connections 2023-03-27T10:22:35.834 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.834+0000 7f7958304700 1 --2- 172.21.15.159:0/2245674882 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f793c09b3a0 0x7f793c09d850 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.834 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.834+0000 7f7958304700 1 --2- 172.21.15.159:0/2245674882 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f7950100810 0x7f795019cec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.834 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.834+0000 7f7958304700 1 --2- 172.21.15.159:0/2245674882 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f79500febd0 0x7f7950068450 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.835 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.834+0000 7f7958304700 1 --2- 172.21.15.159:0/2245674882 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f79500ffe60 0x7f795019a870 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.835 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.834+0000 7f7958304700 1 -- 172.21.15.159:0/2245674882 >> 172.21.15.159:0/2245674882 conn(0x7f79500fa210 msgr2=0x7f79500fc490 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:35.835 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.834+0000 7f7958304700 1 -- 172.21.15.159:0/2245674882 shutdown_connections 2023-03-27T10:22:35.835 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.834+0000 7f7958304700 1 -- 172.21.15.159:0/2245674882 wait complete. 2023-03-27T10:22:35.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:35 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3773497711' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2023-03-27T10:22:35.836 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:35 smithi161 ceph-mon[94333]: osdmap e87: 12 total, 12 up, 12 in 2023-03-27T10:22:35.836 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:35 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/999449511' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]: dispatch 2023-03-27T10:22:35.836 INFO:teuthology.orchestra.run.smithi159.stderr:added data pool 4 to fsmap 2023-03-27T10:22:35.844 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:35 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3773497711' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2023-03-27T10:22:35.844 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:35 smithi159 ceph-mon[93039]: osdmap e87: 12 total, 12 up, 12 in 2023-03-27T10:22:35.845 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:35 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/999449511' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]: dispatch 2023-03-27T10:22:35.845 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-03-27T10:22:35.967 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.968+0000 7fa13ede0700 1 -- 172.21.15.159:0/1684920951 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa1380ff8e0 msgr2=0x7fa1380ffd50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:35.968 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.968+0000 7fa13ede0700 1 --2- 172.21.15.159:0/1684920951 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa1380ff8e0 0x7fa1380ffd50 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7fa13000b0a0 tx=0x7fa13000b3b0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.968 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.968+0000 7fa13ede0700 1 -- 172.21.15.159:0/1684920951 shutdown_connections 2023-03-27T10:22:35.968 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.968+0000 7fa13ede0700 1 --2- 172.21.15.159:0/1684920951 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa1380ff8e0 0x7fa1380ffd50 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.968 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.968+0000 7fa13ede0700 1 --2- 172.21.15.159:0/1684920951 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa1380fe5e0 0x7fa1380fe9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.968 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.968+0000 7fa13ede0700 1 --2- 172.21.15.159:0/1684920951 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa1380fef30 0x7fa1380ff3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.968 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.968+0000 7fa13ede0700 1 -- 172.21.15.159:0/1684920951 >> 172.21.15.159:0/1684920951 conn(0x7fa1380fa1f0 msgr2=0x7fa1380fc660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:35.969 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.968+0000 7fa13ede0700 1 -- 172.21.15.159:0/1684920951 shutdown_connections 2023-03-27T10:22:35.969 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.968+0000 7fa13ede0700 1 -- 172.21.15.159:0/1684920951 wait complete. 2023-03-27T10:22:35.970 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa13ede0700 1 Processor -- start 2023-03-27T10:22:35.971 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa13ede0700 1 -- start start 2023-03-27T10:22:35.971 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa13ede0700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa1380fe5e0 0x7fa138197d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:35.971 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa13ede0700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa1380fef30 0x7fa138198270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:35.971 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa13ede0700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa1380ff8e0 0x7fa13819f2a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:35.971 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa13ede0700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa13819f870 con 0x7fa1380fef30 2023-03-27T10:22:35.971 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa13ede0700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa13819f9e0 con 0x7fa1380ff8e0 2023-03-27T10:22:35.971 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa13ede0700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa13819fba0 con 0x7fa1380fe5e0 2023-03-27T10:22:35.972 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa137fff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa1380fef30 0x7fa138198270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:35.972 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa137fff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa1380fef30 0x7fa138198270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45706/0 (socket says 172.21.15.159:45706) 2023-03-27T10:22:35.972 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa137fff700 1 -- 172.21.15.159:0/3167629187 learned_addr learned my addr 172.21.15.159:0/3167629187 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:35.972 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa13cb7c700 1 --2- 172.21.15.159:0/3167629187 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa1380fe5e0 0x7fa138197d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:35.972 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa137fff700 1 -- 172.21.15.159:0/3167629187 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa1380fe5e0 msgr2=0x7fa138197d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:35.972 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa137fff700 1 --2- 172.21.15.159:0/3167629187 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa1380fe5e0 0x7fa138197d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.972 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa13d37d700 1 --2- 172.21.15.159:0/3167629187 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa1380ff8e0 0x7fa13819f2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:35.972 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa137fff700 1 -- 172.21.15.159:0/3167629187 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa1380ff8e0 msgr2=0x7fa13819f2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:35.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa137fff700 1 --2- 172.21.15.159:0/3167629187 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa1380ff8e0 0x7fa13819f2a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:35.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa137fff700 1 -- 172.21.15.159:0/3167629187 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa130009d00 con 0x7fa1380fef30 2023-03-27T10:22:35.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa137fff700 1 --2- 172.21.15.159:0/3167629187 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa1380fef30 0x7fa138198270 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fa12400e9a0 tx=0x7fa12400ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:35.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.969+0000 7fa135ffb700 1 -- 172.21.15.159:0/3167629187 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa124009790 con 0x7fa1380fef30 2023-03-27T10:22:35.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.970+0000 7fa135ffb700 1 -- 172.21.15.159:0/3167629187 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa12400cbd0 con 0x7fa1380fef30 2023-03-27T10:22:35.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.970+0000 7fa13ede0700 1 -- 172.21.15.159:0/3167629187 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa13810d860 con 0x7fa1380fef30 2023-03-27T10:22:35.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.970+0000 7fa135ffb700 1 -- 172.21.15.159:0/3167629187 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa124018680 con 0x7fa1380fef30 2023-03-27T10:22:35.973 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.970+0000 7fa13ede0700 1 -- 172.21.15.159:0/3167629187 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa13810d6d0 con 0x7fa1380fef30 2023-03-27T10:22:35.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.971+0000 7fa13ede0700 1 -- 172.21.15.159:0/3167629187 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa1381093c0 con 0x7fa1380fef30 2023-03-27T10:22:35.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.972+0000 7fa135ffb700 1 -- 172.21.15.159:0/3167629187 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7fa12400c040 con 0x7fa1380fef30 2023-03-27T10:22:35.974 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.972+0000 7fa135ffb700 1 --2- 172.21.15.159:0/3167629187 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa12809b420 0x7fa12809d8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:35.975 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.972+0000 7fa135ffb700 1 -- 172.21.15.159:0/3167629187 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(89..89 src has 1..89) v4 ==== 8730+0+0 (secure 0 0 0) 0x7fa124014070 con 0x7fa1380fef30 2023-03-27T10:22:35.975 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.972+0000 7fa13cb7c700 1 --2- 172.21.15.159:0/3167629187 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa12809b420 0x7fa12809d8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:35.975 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.973+0000 7fa13cb7c700 1 --2- 172.21.15.159:0/3167629187 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa12809b420 0x7fa12809d8d0 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7fa138199200 tx=0x7fa12c00a380 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:35.975 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:35.976+0000 7fa135ffb700 1 -- 172.21.15.159:0/3167629187 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fa124090d40 con 0x7fa1380fef30 2023-03-27T10:22:36.207 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.207+0000 7fa13ede0700 1 -- 172.21.15.159:0/3167629187 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa138061560 con 0x7fa1380fef30 2023-03-27T10:22:36.207 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.208+0000 7fa135ffb700 1 -- 172.21.15.159:0/3167629187 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) v1 ==== 94+0+10706 (secure 0 0 0) 0x7fa124026070 con 0x7fa1380fef30 2023-03-27T10:22:36.208 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:36.208 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":17,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14556,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6835/3048624466","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6834","nonce":3048624466},{"type":"v1","addr":"172.21.15.159:6835","nonce":3048624466}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14562,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6837/1998956589","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6836","nonce":1998956589},{"type":"v1","addr":"172.21.15.159:6837","nonce":1998956589}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14619,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6839/793152328","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6838","nonce":793152328},{"type":"v1","addr":"172.21.15.159:6839","nonce":793152328}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14631,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6841/536144142","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6840","nonce":536144142},{"type":"v1","addr":"172.21.15.159:6841","nonce":536144142}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24424,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6833/1022778932","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6832","nonce":1022778932},{"type":"v1","addr":"172.21.15.171:6833","nonce":1022778932}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24440,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6833/2969583384","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6832","nonce":2969583384},{"type":"v1","addr":"172.21.15.161:6833","nonce":2969583384}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24464,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6835/761787339","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6834","nonce":761787339},{"type":"v1","addr":"172.21.15.161:6835","nonce":761787339}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24472,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6835/1823903399","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6834","nonce":1823903399},{"type":"v1","addr":"172.21.15.171:6835","nonce":1823903399}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24484,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6837/2737069329","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6836","nonce":2737069329},{"type":"v1","addr":"172.21.15.171:6837","nonce":2737069329}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24491,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6837/406989937","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6836","nonce":406989937},{"type":"v1","addr":"172.21.15.161:6837","nonce":406989937}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9},{"gid":24502,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6839/1538859602","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6838","nonce":1538859602},{"type":"v1","addr":"172.21.15.171:6839","nonce":1538859602}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":10}],"filesystems":[{"mdsmap":{"epoch":17,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2023-03-27T10:22:31.255022+0000","modified":"2023-03-27T10:22:35.829205+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24539},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24539":{"gid":24539,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.161:6839/2491878460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6838","nonce":2491878460},{"type":"v1","addr":"172.21.15.161:6839","nonce":2491878460}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":1},"id":1}]} 2023-03-27T10:22:36.210 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.211+0000 7fa13ede0700 1 -- 172.21.15.159:0/3167629187 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa12809b420 msgr2=0x7fa12809d8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:36.210 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.211+0000 7fa13ede0700 1 --2- 172.21.15.159:0/3167629187 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa12809b420 0x7fa12809d8d0 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7fa138199200 tx=0x7fa12c00a380 comp rx=0 tx=0).stop 2023-03-27T10:22:36.210 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.211+0000 7fa13ede0700 1 -- 172.21.15.159:0/3167629187 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa1380fef30 msgr2=0x7fa138198270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:36.210 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.211+0000 7fa13ede0700 1 --2- 172.21.15.159:0/3167629187 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa1380fef30 0x7fa138198270 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fa12400e9a0 tx=0x7fa12400ecb0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.210 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.211+0000 7fa13ede0700 1 -- 172.21.15.159:0/3167629187 shutdown_connections 2023-03-27T10:22:36.211 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.211+0000 7fa13ede0700 1 --2- 172.21.15.159:0/3167629187 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fa12809b420 0x7fa12809d8d0 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.211 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.211+0000 7fa13ede0700 1 --2- 172.21.15.159:0/3167629187 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fa1380fe5e0 0x7fa138197d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.211 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.211+0000 7fa13ede0700 1 --2- 172.21.15.159:0/3167629187 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fa1380ff8e0 0x7fa13819f2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.211 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.211+0000 7fa13ede0700 1 --2- 172.21.15.159:0/3167629187 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fa1380fef30 0x7fa138198270 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.211 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.211+0000 7fa13ede0700 1 -- 172.21.15.159:0/3167629187 >> 172.21.15.159:0/3167629187 conn(0x7fa1380fa1f0 msgr2=0x7fa1380fbf00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:36.211 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.211+0000 7fa13ede0700 1 -- 172.21.15.159:0/3167629187 shutdown_connections 2023-03-27T10:22:36.211 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.211+0000 7fa13ede0700 1 -- 172.21.15.159:0/3167629187 wait complete. 2023-03-27T10:22:36.212 INFO:teuthology.orchestra.run.smithi159.stderr:dumped fsmap epoch 17 2023-03-27T10:22:36.222 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-03-27T10:22:36.343 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.344+0000 7f107ba3c700 1 -- 172.21.15.159:0/1556617765 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1074102da0 msgr2=0x7f1074105220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:36.344 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.344+0000 7f107ba3c700 1 --2- 172.21.15.159:0/1556617765 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1074102da0 0x7f1074105220 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f1070009990 tx=0x7f1070009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.344 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.344+0000 7f107ba3c700 1 -- 172.21.15.159:0/1556617765 shutdown_connections 2023-03-27T10:22:36.344 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.344+0000 7f107ba3c700 1 --2- 172.21.15.159:0/1556617765 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f10741058b0 0x7f1074107d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.344 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.344+0000 7f107ba3c700 1 --2- 172.21.15.159:0/1556617765 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1074068600 0x7f1074102860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.344 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.344+0000 7f107ba3c700 1 --2- 172.21.15.159:0/1556617765 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1074102da0 0x7f1074105220 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.344 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.344+0000 7f107ba3c700 1 -- 172.21.15.159:0/1556617765 >> 172.21.15.159:0/1556617765 conn(0x7f10740fa210 msgr2=0x7f10740fc620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:36.345 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.344+0000 7f107ba3c700 1 -- 172.21.15.159:0/1556617765 shutdown_connections 2023-03-27T10:22:36.345 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.344+0000 7f107ba3c700 1 -- 172.21.15.159:0/1556617765 wait complete. 2023-03-27T10:22:36.346 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f107ba3c700 1 Processor -- start 2023-03-27T10:22:36.346 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f107ba3c700 1 -- start start 2023-03-27T10:22:36.347 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f107ba3c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1074068600 0x7f1074195a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:36.347 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f107ba3c700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1074102da0 0x7f1074195f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:36.347 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f107ba3c700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f10741058b0 0x7f107419cfe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:36.347 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f107ba3c700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f107419d5b0 con 0x7f1074068600 2023-03-27T10:22:36.347 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f107ba3c700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f107419d6f0 con 0x7f1074102da0 2023-03-27T10:22:36.347 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f107ba3c700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f107419d880 con 0x7f10741058b0 2023-03-27T10:22:36.347 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f10797d8700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1074068600 0x7f1074195a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:36.348 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f10797d8700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1074068600 0x7f1074195a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45718/0 (socket says 172.21.15.159:45718) 2023-03-27T10:22:36.348 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f10797d8700 1 -- 172.21.15.159:0/81782325 learned_addr learned my addr 172.21.15.159:0/81782325 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:36.348 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f1078fd7700 1 --2- 172.21.15.159:0/81782325 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1074102da0 0x7f1074195f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:36.348 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f1079fd9700 1 --2- 172.21.15.159:0/81782325 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f10741058b0 0x7f107419cfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:36.348 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f10797d8700 1 -- 172.21.15.159:0/81782325 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f10741058b0 msgr2=0x7f107419cfe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:36.348 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f10797d8700 1 --2- 172.21.15.159:0/81782325 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f10741058b0 0x7f107419cfe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.348 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f10797d8700 1 -- 172.21.15.159:0/81782325 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1074102da0 msgr2=0x7f1074195f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:36.348 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f10797d8700 1 --2- 172.21.15.159:0/81782325 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1074102da0 0x7f1074195f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.349 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f10797d8700 1 -- 172.21.15.159:0/81782325 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f10700095f0 con 0x7f1074068600 2023-03-27T10:22:36.349 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f1078fd7700 1 --2- 172.21.15.159:0/81782325 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1074102da0 0x7f1074195f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:22:36.349 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f10797d8700 1 --2- 172.21.15.159:0/81782325 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1074068600 0x7f1074195a30 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f106800d950 tx=0x7f106800dd10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:36.349 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f10667fc700 1 -- 172.21.15.159:0/81782325 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f106800bb60 con 0x7f1074068600 2023-03-27T10:22:36.349 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f10667fc700 1 -- 172.21.15.159:0/81782325 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1068007620 con 0x7f1074068600 2023-03-27T10:22:36.349 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f107ba3c700 1 -- 172.21.15.159:0/81782325 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f107410b650 con 0x7f1074068600 2023-03-27T10:22:36.349 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.345+0000 7f10667fc700 1 -- 172.21.15.159:0/81782325 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1068005560 con 0x7f1074068600 2023-03-27T10:22:36.350 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.346+0000 7f107ba3c700 1 -- 172.21.15.159:0/81782325 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f107410b4c0 con 0x7f1074068600 2023-03-27T10:22:36.350 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.347+0000 7f107ba3c700 1 -- 172.21.15.159:0/81782325 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f10740fbd80 con 0x7f1074068600 2023-03-27T10:22:36.350 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.348+0000 7f10667fc700 1 -- 172.21.15.159:0/81782325 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f106800c040 con 0x7f1074068600 2023-03-27T10:22:36.350 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.348+0000 7f10667fc700 1 --2- 172.21.15.159:0/81782325 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f106009b420 0x7f106009d8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:36.351 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.348+0000 7f1078fd7700 1 --2- 172.21.15.159:0/81782325 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f106009b420 0x7f106009d8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:36.351 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.348+0000 7f10667fc700 1 -- 172.21.15.159:0/81782325 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(89..89 src has 1..89) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f10680c5460 con 0x7f1074068600 2023-03-27T10:22:36.351 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.349+0000 7f1078fd7700 1 --2- 172.21.15.159:0/81782325 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f106009b420 0x7f106009d8d0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f1070006010 tx=0x7f107000b540 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:36.351 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.352+0000 7f10667fc700 1 -- 172.21.15.159:0/81782325 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f1068090b90 con 0x7f1074068600 2023-03-27T10:22:36.538 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.539+0000 7f107ba3c700 1 -- 172.21.15.159:0/81782325 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f107402cdb0 con 0x7f1074068600 2023-03-27T10:22:36.539 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.540+0000 7f10667fc700 1 -- 172.21.15.159:0/81782325 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) v1 ==== 74+0+23993 (secure 0 0 0) 0x7f1068016020 con 0x7f1074068600 2023-03-27T10:22:36.540 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:36.540 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":89,"fsid":"2c3cd140-cc88-11ed-9afe-001a4aab830c","created":"2023-03-27T10:15:32.617230+0000","modified":"2023-03-27T10:22:35.828925+0000","last_up_change":"2023-03-27T10:21:49.442480+0000","last_in_change":"2023-03-27T10:21:32.265278+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-03-27T10:18:35.277335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-03-27T10:22:29.479987+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-03-27T10:22:30.424727+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-03-27T10:22:33.650889+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"f47f99b9-47f9-4d94-83d9-33b1278be17e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6803","nonce":87273908}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6805","nonce":87273908}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6809","nonce":87273908}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6807","nonce":87273908}]},"public_addr":"172.21.15.159:6803/87273908","cluster_addr":"172.21.15.159:6805/87273908","heartbeat_back_addr":"172.21.15.159:6809/87273908","heartbeat_front_addr":"172.21.15.159:6807/87273908","state":["exists","up"]},{"osd":1,"uuid":"97f66da9-22b4-4eb5-9c77-9d37cf0fca82","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6800","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6801","nonce":2094166586}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6802","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6803","nonce":2094166586}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6806","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6807","nonce":2094166586}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6804","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6805","nonce":2094166586}]},"public_addr":"172.21.15.161:6801/2094166586","cluster_addr":"172.21.15.161:6803/2094166586","heartbeat_back_addr":"172.21.15.161:6807/2094166586","heartbeat_front_addr":"172.21.15.161:6805/2094166586","state":["exists","up"]},{"osd":2,"uuid":"ba9f5f82-3aef-42f7-a9e6-0e07aa7d8df1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6801","nonce":2724827688}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6803","nonce":2724827688}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6807","nonce":2724827688}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6805","nonce":2724827688}]},"public_addr":"172.21.15.171:6801/2724827688","cluster_addr":"172.21.15.171:6803/2724827688","heartbeat_back_addr":"172.21.15.171:6807/2724827688","heartbeat_front_addr":"172.21.15.171:6805/2724827688","state":["exists","up"]},{"osd":3,"uuid":"4ef4b519-fdef-4d38-b387-bc14ea381bd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6811","nonce":1992276871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6813","nonce":1992276871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6817","nonce":1992276871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6815","nonce":1992276871}]},"public_addr":"172.21.15.159:6811/1992276871","cluster_addr":"172.21.15.159:6813/1992276871","heartbeat_back_addr":"172.21.15.159:6817/1992276871","heartbeat_front_addr":"172.21.15.159:6815/1992276871","state":["exists","up"]},{"osd":4,"uuid":"982675c1-dd87-46b4-97ef-a5dde1b1d777","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6808","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6809","nonce":3978602555}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6810","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6811","nonce":3978602555}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6814","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6815","nonce":3978602555}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6812","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6813","nonce":3978602555}]},"public_addr":"172.21.15.161:6809/3978602555","cluster_addr":"172.21.15.161:6811/3978602555","heartbeat_back_addr":"172.21.15.161:6815/3978602555","heartbeat_front_addr":"172.21.15.161:6813/3978602555","state":["exists","up"]},{"osd":5,"uuid":"b4de4a85-4f91-4194-a1ce-db12f1898c4f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6809","nonce":569611891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6811","nonce":569611891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6815","nonce":569611891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6813","nonce":569611891}]},"public_addr":"172.21.15.171:6809/569611891","cluster_addr":"172.21.15.171:6811/569611891","heartbeat_back_addr":"172.21.15.171:6815/569611891","heartbeat_front_addr":"172.21.15.171:6813/569611891","state":["exists","up"]},{"osd":6,"uuid":"dff7d5f4-655f-4baa-9084-03591a4b3bb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6819","nonce":1176978350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6821","nonce":1176978350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6825","nonce":1176978350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6823","nonce":1176978350}]},"public_addr":"172.21.15.159:6819/1176978350","cluster_addr":"172.21.15.159:6821/1176978350","heartbeat_back_addr":"172.21.15.159:6825/1176978350","heartbeat_front_addr":"172.21.15.159:6823/1176978350","state":["exists","up"]},{"osd":7,"uuid":"30f026d9-6963-4468-95e8-78c7ed2e89f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6816","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6817","nonce":2098174471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6818","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6819","nonce":2098174471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6822","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6823","nonce":2098174471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6820","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6821","nonce":2098174471}]},"public_addr":"172.21.15.161:6817/2098174471","cluster_addr":"172.21.15.161:6819/2098174471","heartbeat_back_addr":"172.21.15.161:6823/2098174471","heartbeat_front_addr":"172.21.15.161:6821/2098174471","state":["exists","up"]},{"osd":8,"uuid":"9f69fbcc-32ff-4d4b-b562-ee0a12745cc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6817","nonce":1033586906}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6819","nonce":1033586906}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6823","nonce":1033586906}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6821","nonce":1033586906}]},"public_addr":"172.21.15.171:6817/1033586906","cluster_addr":"172.21.15.171:6819/1033586906","heartbeat_back_addr":"172.21.15.171:6823/1033586906","heartbeat_front_addr":"172.21.15.171:6821/1033586906","state":["exists","up"]},{"osd":9,"uuid":"d7fb9eed-901b-4943-9529-6edcba2385e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6827","nonce":3285179829}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6829","nonce":3285179829}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6832","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6833","nonce":3285179829}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6831","nonce":3285179829}]},"public_addr":"172.21.15.159:6827/3285179829","cluster_addr":"172.21.15.159:6829/3285179829","heartbeat_back_addr":"172.21.15.159:6833/3285179829","heartbeat_front_addr":"172.21.15.159:6831/3285179829","state":["exists","up"]},{"osd":10,"uuid":"4871ca03-30ae-4e87-942c-9441dd31fb40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":83,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6824","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6825","nonce":946677932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6826","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6827","nonce":946677932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6830","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6831","nonce":946677932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6828","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6829","nonce":946677932}]},"public_addr":"172.21.15.161:6825/946677932","cluster_addr":"172.21.15.161:6827/946677932","heartbeat_back_addr":"172.21.15.161:6831/946677932","heartbeat_front_addr":"172.21.15.161:6829/946677932","state":["exists","up"]},{"osd":11,"uuid":"1fb37d9a-ed52-4c0d-a082-65e17d242106","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6825","nonce":596725174}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6827","nonce":596725174}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6831","nonce":596725174}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6829","nonce":596725174}]},"public_addr":"172.21.15.171:6825/596725174","cluster_addr":"172.21.15.171:6827/596725174","heartbeat_back_addr":"172.21.15.171:6831/596725174","heartbeat_front_addr":"172.21.15.171:6829/596725174","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:17:49.128592+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:10.944593+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:32.989432+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:54.675294+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:16.543485+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:38.390326+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:59.566202+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:20:21.348420+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:20:43.383576+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:04.710992+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:26.286573+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:48.277317+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.159:0/653469093":"2023-03-28T10:16:26.900382+0000","172.21.15.159:6800/3561797228":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/3045586257":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/2859804517":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6801/3561797228":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/3487813906":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/1306320900":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6800/1729835725":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6801/1729835725":"2023-03-28T10:16:04.737100+0000","172.21.15.159:0/72206061":"2023-03-28T10:16:04.737100+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-03-27T10:22:36.542 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.542+0000 7f107ba3c700 1 -- 172.21.15.159:0/81782325 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f106009b420 msgr2=0x7f106009d8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:36.542 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.542+0000 7f107ba3c700 1 --2- 172.21.15.159:0/81782325 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f106009b420 0x7f106009d8d0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f1070006010 tx=0x7f107000b540 comp rx=0 tx=0).stop 2023-03-27T10:22:36.542 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.542+0000 7f107ba3c700 1 -- 172.21.15.159:0/81782325 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1074068600 msgr2=0x7f1074195a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:36.542 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.542+0000 7f107ba3c700 1 --2- 172.21.15.159:0/81782325 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1074068600 0x7f1074195a30 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f106800d950 tx=0x7f106800dd10 comp rx=0 tx=0).stop 2023-03-27T10:22:36.542 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.542+0000 7f107ba3c700 1 -- 172.21.15.159:0/81782325 shutdown_connections 2023-03-27T10:22:36.542 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.542+0000 7f107ba3c700 1 --2- 172.21.15.159:0/81782325 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f106009b420 0x7f106009d8d0 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.542 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.542+0000 7f107ba3c700 1 --2- 172.21.15.159:0/81782325 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f10741058b0 0x7f107419cfe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.543 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.542+0000 7f107ba3c700 1 --2- 172.21.15.159:0/81782325 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1074068600 0x7f1074195a30 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.543 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.542+0000 7f107ba3c700 1 --2- 172.21.15.159:0/81782325 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1074102da0 0x7f1074195f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.543 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.542+0000 7f107ba3c700 1 -- 172.21.15.159:0/81782325 >> 172.21.15.159:0/81782325 conn(0x7f10740fa210 msgr2=0x7f10741010c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:36.543 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.542+0000 7f107ba3c700 1 -- 172.21.15.159:0/81782325 shutdown_connections 2023-03-27T10:22:36.543 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.542+0000 7f107ba3c700 1 -- 172.21.15.159:0/81782325 wait complete. 2023-03-27T10:22:36.553 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:36 smithi159 ceph-mon[93039]: pgmap v261: 129 pgs: 25 creating+peering, 74 active+clean, 30 unknown; 579 KiB data, 76 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s wr, 10 op/s 2023-03-27T10:22:36.553 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:36 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/999449511' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]': finished 2023-03-27T10:22:36.553 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:36 smithi159 ceph-mon[93039]: osdmap e88: 12 total, 12 up, 12 in 2023-03-27T10:22:36.553 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:36 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/2245674882' entity='client.admin' cmd=[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]: dispatch 2023-03-27T10:22:36.553 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:36 smithi159 ceph-mon[93039]: osdmap e89: 12 total, 12 up, 12 in 2023-03-27T10:22:36.553 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:36 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/2245674882' entity='client.admin' cmd='[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]': finished 2023-03-27T10:22:36.554 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:36 smithi159 ceph-mon[93039]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-03-27T10:22:36.554 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:36 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3167629187' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:36.555 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-03-27T10:22:36.677 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 -- 172.21.15.159:0/2292494906 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f77e40fec00 msgr2=0x7f77e40ff010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:36.678 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 --2- 172.21.15.159:0/2292494906 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f77e40fec00 0x7f77e40ff010 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f77e0009990 tx=0x7f77e0009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.678 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 -- 172.21.15.159:0/2292494906 shutdown_connections 2023-03-27T10:22:36.678 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 --2- 172.21.15.159:0/2292494906 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f77e4100780 0x7f77e4107040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.678 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 --2- 172.21.15.159:0/2292494906 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f77e40fec00 0x7f77e40ff010 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.678 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 --2- 172.21.15.159:0/2292494906 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f77e40ffdf0 0x7f77e4100240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.678 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 -- 172.21.15.159:0/2292494906 >> 172.21.15.159:0/2292494906 conn(0x7f77e40fa1e0 msgr2=0x7f77e40fc650 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:36.678 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 -- 172.21.15.159:0/2292494906 shutdown_connections 2023-03-27T10:22:36.679 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 -- 172.21.15.159:0/2292494906 wait complete. 2023-03-27T10:22:36.679 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 Processor -- start 2023-03-27T10:22:36.679 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 -- start start 2023-03-27T10:22:36.680 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f77e40fec00 0x7f77e4069470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:36.680 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f77e40ffdf0 0x7f77e419a7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:36.681 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f77e4100780 0x7f77e419ce00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:36.681 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f77e419d5a0 con 0x7f77e40ffdf0 2023-03-27T10:22:36.681 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f77e419d710 con 0x7f77e40fec00 2023-03-27T10:22:36.681 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.678+0000 7f77eb7d2700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f77e419d8d0 con 0x7f77e4100780 2023-03-27T10:22:36.681 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.679+0000 7f77e8d6d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f77e40ffdf0 0x7f77e419a7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:36.681 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.679+0000 7f77e8d6d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f77e40ffdf0 0x7f77e419a7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45734/0 (socket says 172.21.15.159:45734) 2023-03-27T10:22:36.681 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.679+0000 7f77e8d6d700 1 -- 172.21.15.159:0/3317798849 learned_addr learned my addr 172.21.15.159:0/3317798849 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:36.682 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.679+0000 7f77e956e700 1 --2- 172.21.15.159:0/3317798849 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f77e40fec00 0x7f77e4069470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:36.682 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.679+0000 7f77e9d6f700 1 --2- 172.21.15.159:0/3317798849 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f77e4100780 0x7f77e419ce00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:36.682 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.679+0000 7f77e8d6d700 1 -- 172.21.15.159:0/3317798849 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f77e4100780 msgr2=0x7f77e419ce00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:36.682 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.679+0000 7f77e8d6d700 1 --2- 172.21.15.159:0/3317798849 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f77e4100780 0x7f77e419ce00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.682 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.679+0000 7f77e8d6d700 1 -- 172.21.15.159:0/3317798849 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f77e40fec00 msgr2=0x7f77e4069470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:36.682 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.679+0000 7f77e8d6d700 1 --2- 172.21.15.159:0/3317798849 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f77e40fec00 0x7f77e4069470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.682 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.679+0000 7f77e8d6d700 1 -- 172.21.15.159:0/3317798849 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f77e00095f0 con 0x7f77e40ffdf0 2023-03-27T10:22:36.682 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.679+0000 7f77e8d6d700 1 --2- 172.21.15.159:0/3317798849 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f77e40ffdf0 0x7f77e419a7c0 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f77d400e9a0 tx=0x7f77d400ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:36.683 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.679+0000 7f77da7fc700 1 -- 172.21.15.159:0/3317798849 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f77d4009790 con 0x7f77e40ffdf0 2023-03-27T10:22:36.683 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.679+0000 7f77eb7d2700 1 -- 172.21.15.159:0/3317798849 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f77e410a560 con 0x7f77e40ffdf0 2023-03-27T10:22:36.683 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.679+0000 7f77da7fc700 1 -- 172.21.15.159:0/3317798849 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f77d400cbd0 con 0x7f77e40ffdf0 2023-03-27T10:22:36.683 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.679+0000 7f77da7fc700 1 -- 172.21.15.159:0/3317798849 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f77d4018680 con 0x7f77e40ffdf0 2023-03-27T10:22:36.683 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.679+0000 7f77eb7d2700 1 -- 172.21.15.159:0/3317798849 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f77e410a3d0 con 0x7f77e40ffdf0 2023-03-27T10:22:36.683 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.680+0000 7f77eb7d2700 1 -- 172.21.15.159:0/3317798849 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f77e4106190 con 0x7f77e40ffdf0 2023-03-27T10:22:36.683 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.681+0000 7f77da7fc700 1 -- 172.21.15.159:0/3317798849 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f77d400c040 con 0x7f77e40ffdf0 2023-03-27T10:22:36.684 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.682+0000 7f77da7fc700 1 --2- 172.21.15.159:0/3317798849 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f77d009f7b0 0x7f77d00a1c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:36.684 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.682+0000 7f77e956e700 1 --2- 172.21.15.159:0/3317798849 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f77d009f7b0 0x7f77d00a1c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:36.684 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.682+0000 7f77da7fc700 1 -- 172.21.15.159:0/3317798849 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(89..89 src has 1..89) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f77d4014070 con 0x7f77e40ffdf0 2023-03-27T10:22:36.684 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.682+0000 7f77e956e700 1 --2- 172.21.15.159:0/3317798849 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f77d009f7b0 0x7f77d00a1c60 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f77e0006010 tx=0x7f77e0005960 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:36.685 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.685+0000 7f77da7fc700 1 -- 172.21.15.159:0/3317798849 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f77d4090e00 con 0x7f77e40ffdf0 2023-03-27T10:22:36.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:36 smithi171 ceph-mon[94808]: pgmap v261: 129 pgs: 25 creating+peering, 74 active+clean, 30 unknown; 579 KiB data, 76 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s wr, 10 op/s 2023-03-27T10:22:36.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:36 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/999449511' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]': finished 2023-03-27T10:22:36.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:36 smithi171 ceph-mon[94808]: osdmap e88: 12 total, 12 up, 12 in 2023-03-27T10:22:36.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:36 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/2245674882' entity='client.admin' cmd=[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]: dispatch 2023-03-27T10:22:36.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:36 smithi171 ceph-mon[94808]: osdmap e89: 12 total, 12 up, 12 in 2023-03-27T10:22:36.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:36 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/2245674882' entity='client.admin' cmd='[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]': finished 2023-03-27T10:22:36.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:36 smithi171 ceph-mon[94808]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-03-27T10:22:36.727 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:36 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3167629187' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:36.834 INFO:journalctl@ceph.mds.b.smithi161.stdout:Mar 27 10:22:36 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-b[119432]: 2023-03-27T10:22:36.544+0000 7f3362a84700 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2023-03-27T10:22:36.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:36 smithi161 ceph-mon[94333]: pgmap v261: 129 pgs: 25 creating+peering, 74 active+clean, 30 unknown; 579 KiB data, 76 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s wr, 10 op/s 2023-03-27T10:22:36.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:36 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/999449511' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]': finished 2023-03-27T10:22:36.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:36 smithi161 ceph-mon[94333]: osdmap e88: 12 total, 12 up, 12 in 2023-03-27T10:22:36.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:36 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/2245674882' entity='client.admin' cmd=[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]: dispatch 2023-03-27T10:22:36.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:36 smithi161 ceph-mon[94333]: osdmap e89: 12 total, 12 up, 12 in 2023-03-27T10:22:36.835 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:36 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/2245674882' entity='client.admin' cmd='[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]': finished 2023-03-27T10:22:36.836 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:36 smithi161 ceph-mon[94333]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-03-27T10:22:36.836 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:36 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3167629187' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:36.871 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.872+0000 7f77eb7d2700 1 -- 172.21.15.159:0/3317798849 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f77e4066c50 con 0x7f77e40ffdf0 2023-03-27T10:22:36.873 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.874+0000 7f77da7fc700 1 -- 172.21.15.159:0/3317798849 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) v1 ==== 74+0+23993 (secure 0 0 0) 0x7f77d4026070 con 0x7f77e40ffdf0 2023-03-27T10:22:36.873 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:36.874 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":89,"fsid":"2c3cd140-cc88-11ed-9afe-001a4aab830c","created":"2023-03-27T10:15:32.617230+0000","modified":"2023-03-27T10:22:35.828925+0000","last_up_change":"2023-03-27T10:21:49.442480+0000","last_in_change":"2023-03-27T10:21:32.265278+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-03-27T10:18:35.277335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-03-27T10:22:29.479987+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-03-27T10:22:30.424727+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-03-27T10:22:33.650889+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"f47f99b9-47f9-4d94-83d9-33b1278be17e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6803","nonce":87273908}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6805","nonce":87273908}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6809","nonce":87273908}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6807","nonce":87273908}]},"public_addr":"172.21.15.159:6803/87273908","cluster_addr":"172.21.15.159:6805/87273908","heartbeat_back_addr":"172.21.15.159:6809/87273908","heartbeat_front_addr":"172.21.15.159:6807/87273908","state":["exists","up"]},{"osd":1,"uuid":"97f66da9-22b4-4eb5-9c77-9d37cf0fca82","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6800","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6801","nonce":2094166586}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6802","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6803","nonce":2094166586}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6806","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6807","nonce":2094166586}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6804","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6805","nonce":2094166586}]},"public_addr":"172.21.15.161:6801/2094166586","cluster_addr":"172.21.15.161:6803/2094166586","heartbeat_back_addr":"172.21.15.161:6807/2094166586","heartbeat_front_addr":"172.21.15.161:6805/2094166586","state":["exists","up"]},{"osd":2,"uuid":"ba9f5f82-3aef-42f7-a9e6-0e07aa7d8df1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6801","nonce":2724827688}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6803","nonce":2724827688}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6807","nonce":2724827688}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6805","nonce":2724827688}]},"public_addr":"172.21.15.171:6801/2724827688","cluster_addr":"172.21.15.171:6803/2724827688","heartbeat_back_addr":"172.21.15.171:6807/2724827688","heartbeat_front_addr":"172.21.15.171:6805/2724827688","state":["exists","up"]},{"osd":3,"uuid":"4ef4b519-fdef-4d38-b387-bc14ea381bd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6811","nonce":1992276871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6813","nonce":1992276871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6817","nonce":1992276871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6815","nonce":1992276871}]},"public_addr":"172.21.15.159:6811/1992276871","cluster_addr":"172.21.15.159:6813/1992276871","heartbeat_back_addr":"172.21.15.159:6817/1992276871","heartbeat_front_addr":"172.21.15.159:6815/1992276871","state":["exists","up"]},{"osd":4,"uuid":"982675c1-dd87-46b4-97ef-a5dde1b1d777","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6808","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6809","nonce":3978602555}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6810","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6811","nonce":3978602555}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6814","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6815","nonce":3978602555}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6812","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6813","nonce":3978602555}]},"public_addr":"172.21.15.161:6809/3978602555","cluster_addr":"172.21.15.161:6811/3978602555","heartbeat_back_addr":"172.21.15.161:6815/3978602555","heartbeat_front_addr":"172.21.15.161:6813/3978602555","state":["exists","up"]},{"osd":5,"uuid":"b4de4a85-4f91-4194-a1ce-db12f1898c4f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6809","nonce":569611891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6811","nonce":569611891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6815","nonce":569611891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6813","nonce":569611891}]},"public_addr":"172.21.15.171:6809/569611891","cluster_addr":"172.21.15.171:6811/569611891","heartbeat_back_addr":"172.21.15.171:6815/569611891","heartbeat_front_addr":"172.21.15.171:6813/569611891","state":["exists","up"]},{"osd":6,"uuid":"dff7d5f4-655f-4baa-9084-03591a4b3bb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6819","nonce":1176978350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6821","nonce":1176978350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6825","nonce":1176978350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6823","nonce":1176978350}]},"public_addr":"172.21.15.159:6819/1176978350","cluster_addr":"172.21.15.159:6821/1176978350","heartbeat_back_addr":"172.21.15.159:6825/1176978350","heartbeat_front_addr":"172.21.15.159:6823/1176978350","state":["exists","up"]},{"osd":7,"uuid":"30f026d9-6963-4468-95e8-78c7ed2e89f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6816","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6817","nonce":2098174471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6818","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6819","nonce":2098174471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6822","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6823","nonce":2098174471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6820","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6821","nonce":2098174471}]},"public_addr":"172.21.15.161:6817/2098174471","cluster_addr":"172.21.15.161:6819/2098174471","heartbeat_back_addr":"172.21.15.161:6823/2098174471","heartbeat_front_addr":"172.21.15.161:6821/2098174471","state":["exists","up"]},{"osd":8,"uuid":"9f69fbcc-32ff-4d4b-b562-ee0a12745cc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6817","nonce":1033586906}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6819","nonce":1033586906}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6823","nonce":1033586906}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6821","nonce":1033586906}]},"public_addr":"172.21.15.171:6817/1033586906","cluster_addr":"172.21.15.171:6819/1033586906","heartbeat_back_addr":"172.21.15.171:6823/1033586906","heartbeat_front_addr":"172.21.15.171:6821/1033586906","state":["exists","up"]},{"osd":9,"uuid":"d7fb9eed-901b-4943-9529-6edcba2385e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6827","nonce":3285179829}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6829","nonce":3285179829}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6832","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6833","nonce":3285179829}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6831","nonce":3285179829}]},"public_addr":"172.21.15.159:6827/3285179829","cluster_addr":"172.21.15.159:6829/3285179829","heartbeat_back_addr":"172.21.15.159:6833/3285179829","heartbeat_front_addr":"172.21.15.159:6831/3285179829","state":["exists","up"]},{"osd":10,"uuid":"4871ca03-30ae-4e87-942c-9441dd31fb40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":83,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6824","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6825","nonce":946677932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6826","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6827","nonce":946677932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6830","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6831","nonce":946677932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6828","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6829","nonce":946677932}]},"public_addr":"172.21.15.161:6825/946677932","cluster_addr":"172.21.15.161:6827/946677932","heartbeat_back_addr":"172.21.15.161:6831/946677932","heartbeat_front_addr":"172.21.15.161:6829/946677932","state":["exists","up"]},{"osd":11,"uuid":"1fb37d9a-ed52-4c0d-a082-65e17d242106","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6825","nonce":596725174}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6827","nonce":596725174}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6831","nonce":596725174}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6829","nonce":596725174}]},"public_addr":"172.21.15.171:6825/596725174","cluster_addr":"172.21.15.171:6827/596725174","heartbeat_back_addr":"172.21.15.171:6831/596725174","heartbeat_front_addr":"172.21.15.171:6829/596725174","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:17:49.128592+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:10.944593+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:32.989432+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:54.675294+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:16.543485+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:38.390326+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:59.566202+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:20:21.348420+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:20:43.383576+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:04.710992+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:26.286573+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:48.277317+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.159:0/653469093":"2023-03-28T10:16:26.900382+0000","172.21.15.159:6800/3561797228":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/3045586257":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/2859804517":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6801/3561797228":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/3487813906":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/1306320900":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6800/1729835725":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6801/1729835725":"2023-03-28T10:16:04.737100+0000","172.21.15.159:0/72206061":"2023-03-28T10:16:04.737100+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-03-27T10:22:36.875 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.876+0000 7f77eb7d2700 1 -- 172.21.15.159:0/3317798849 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f77d009f7b0 msgr2=0x7f77d00a1c60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:36.875 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.876+0000 7f77eb7d2700 1 --2- 172.21.15.159:0/3317798849 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f77d009f7b0 0x7f77d00a1c60 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f77e0006010 tx=0x7f77e0005960 comp rx=0 tx=0).stop 2023-03-27T10:22:36.875 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.876+0000 7f77eb7d2700 1 -- 172.21.15.159:0/3317798849 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f77e40ffdf0 msgr2=0x7f77e419a7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:36.876 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.876+0000 7f77eb7d2700 1 --2- 172.21.15.159:0/3317798849 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f77e40ffdf0 0x7f77e419a7c0 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f77d400e9a0 tx=0x7f77d400ecb0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.876 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.876+0000 7f77eb7d2700 1 -- 172.21.15.159:0/3317798849 shutdown_connections 2023-03-27T10:22:36.876 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.876+0000 7f77eb7d2700 1 --2- 172.21.15.159:0/3317798849 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f77d009f7b0 0x7f77d00a1c60 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.876 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.876+0000 7f77eb7d2700 1 --2- 172.21.15.159:0/3317798849 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f77e4100780 0x7f77e419ce00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.876 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.876+0000 7f77eb7d2700 1 --2- 172.21.15.159:0/3317798849 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f77e40fec00 0x7f77e4069470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.876 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.876+0000 7f77eb7d2700 1 --2- 172.21.15.159:0/3317798849 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f77e40ffdf0 0x7f77e419a7c0 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:36.876 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.876+0000 7f77eb7d2700 1 -- 172.21.15.159:0/3317798849 >> 172.21.15.159:0/3317798849 conn(0x7f77e40fa1e0 msgr2=0x7f77e41032d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:36.877 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.876+0000 7f77eb7d2700 1 -- 172.21.15.159:0/3317798849 shutdown_connections 2023-03-27T10:22:36.877 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:36.876+0000 7f77eb7d2700 1 -- 172.21.15.159:0/3317798849 wait complete. 2023-03-27T10:22:36.904 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-03-27T10:22:37.027 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.027+0000 7fec5e681700 1 -- 172.21.15.159:0/3486942962 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fec58100810 msgr2=0x7fec581070b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:37.027 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.027+0000 7fec5e681700 1 --2- 172.21.15.159:0/3486942962 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fec58100810 0x7fec581070b0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7fec5000b0a0 tx=0x7fec5000b3b0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.027 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.027+0000 7fec5e681700 1 -- 172.21.15.159:0/3486942962 shutdown_connections 2023-03-27T10:22:37.027 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.027+0000 7fec5e681700 1 --2- 172.21.15.159:0/3486942962 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fec58100810 0x7fec581070b0 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.027 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.027+0000 7fec5e681700 1 --2- 172.21.15.159:0/3486942962 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fec580febd0 0x7fec580fefe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.027 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.027+0000 7fec5e681700 1 --2- 172.21.15.159:0/3486942962 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fec580ffe60 0x7fec581002d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.028 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.027+0000 7fec5e681700 1 -- 172.21.15.159:0/3486942962 >> 172.21.15.159:0/3486942962 conn(0x7fec580fa210 msgr2=0x7fec580fc620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:37.028 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.027+0000 7fec5e681700 1 -- 172.21.15.159:0/3486942962 shutdown_connections 2023-03-27T10:22:37.028 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.027+0000 7fec5e681700 1 -- 172.21.15.159:0/3486942962 wait complete. 2023-03-27T10:22:37.030 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.028+0000 7fec5e681700 1 Processor -- start 2023-03-27T10:22:37.030 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.028+0000 7fec5e681700 1 -- start start 2023-03-27T10:22:37.030 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.028+0000 7fec5e681700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fec580febd0 0x7fec58197dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.030 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.028+0000 7fec5e681700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fec580ffe60 0x7fec58198300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.030 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.028+0000 7fec5e681700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fec58100810 0x7fec5819f330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.030 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.028+0000 7fec5e681700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec5819f900 con 0x7fec580febd0 2023-03-27T10:22:37.030 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.028+0000 7fec5e681700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec5819fa70 con 0x7fec58100810 2023-03-27T10:22:37.031 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.028+0000 7fec5e681700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec5819fc30 con 0x7fec580ffe60 2023-03-27T10:22:37.031 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.028+0000 7fec57fff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fec580febd0 0x7fec58197dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.031 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.028+0000 7fec57fff700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fec580febd0 0x7fec58197dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45744/0 (socket says 172.21.15.159:45744) 2023-03-27T10:22:37.031 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.028+0000 7fec57fff700 1 -- 172.21.15.159:0/320352801 learned_addr learned my addr 172.21.15.159:0/320352801 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:37.031 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.028+0000 7fec577fe700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fec580ffe60 0x7fec58198300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.031 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.028+0000 7fec5cc1e700 1 --2- 172.21.15.159:0/320352801 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fec58100810 0x7fec5819f330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.031 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.029+0000 7fec57fff700 1 -- 172.21.15.159:0/320352801 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fec580ffe60 msgr2=0x7fec58198300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:37.031 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.029+0000 7fec57fff700 1 --2- 172.21.15.159:0/320352801 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fec580ffe60 0x7fec58198300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.032 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.029+0000 7fec57fff700 1 -- 172.21.15.159:0/320352801 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fec58100810 msgr2=0x7fec5819f330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:37.032 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.029+0000 7fec57fff700 1 --2- 172.21.15.159:0/320352801 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fec58100810 0x7fec5819f330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.032 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.029+0000 7fec57fff700 1 -- 172.21.15.159:0/320352801 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fec50009d00 con 0x7fec580febd0 2023-03-27T10:22:37.032 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.029+0000 7fec57fff700 1 --2- 172.21.15.159:0/320352801 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fec580febd0 0x7fec58197dc0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7fec4800b530 tx=0x7fec4800b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:37.032 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.029+0000 7fec557fa700 1 -- 172.21.15.159:0/320352801 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec48011830 con 0x7fec580febd0 2023-03-27T10:22:37.032 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.029+0000 7fec5e681700 1 -- 172.21.15.159:0/320352801 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fec581094e0 con 0x7fec580febd0 2023-03-27T10:22:37.032 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.029+0000 7fec557fa700 1 -- 172.21.15.159:0/320352801 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fec48011e50 con 0x7fec580febd0 2023-03-27T10:22:37.033 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.029+0000 7fec557fa700 1 -- 172.21.15.159:0/320352801 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec4800f2e0 con 0x7fec580febd0 2023-03-27T10:22:37.033 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.029+0000 7fec5e681700 1 -- 172.21.15.159:0/320352801 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fec58109350 con 0x7fec580febd0 2023-03-27T10:22:37.033 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.030+0000 7fec5e681700 1 -- 172.21.15.159:0/320352801 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fec58105ba0 con 0x7fec580febd0 2023-03-27T10:22:37.033 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.031+0000 7fec557fa700 1 -- 172.21.15.159:0/320352801 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7fec48021070 con 0x7fec580febd0 2023-03-27T10:22:37.034 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.031+0000 7fec557fa700 1 --2- 172.21.15.159:0/320352801 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fec4009b400 0x7fec4009d8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.034 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.032+0000 7fec577fe700 1 --2- 172.21.15.159:0/320352801 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fec4009b400 0x7fec4009d8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.034 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.032+0000 7fec557fa700 1 -- 172.21.15.159:0/320352801 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(89..89 src has 1..89) v4 ==== 8730+0+0 (secure 0 0 0) 0x7fec480c49f0 con 0x7fec580febd0 2023-03-27T10:22:37.034 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.032+0000 7fec577fe700 1 --2- 172.21.15.159:0/320352801 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fec4009b400 0x7fec4009d8b0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7fec4c009c60 tx=0x7fec4c009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:37.034 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.035+0000 7fec557fa700 1 -- 172.21.15.159:0/320352801 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fec480900c0 con 0x7fec580febd0 2023-03-27T10:22:37.261 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.262+0000 7fec5e681700 1 -- 172.21.15.159:0/320352801 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fec5802cdb0 con 0x7fec580febd0 2023-03-27T10:22:37.262 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.263+0000 7fec557fa700 1 -- 172.21.15.159:0/320352801 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 17 v17) v1 ==== 94+0+10706 (secure 0 0 0) 0x7fec48015070 con 0x7fec580febd0 2023-03-27T10:22:37.262 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:37.262 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":17,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14556,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6835/3048624466","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6834","nonce":3048624466},{"type":"v1","addr":"172.21.15.159:6835","nonce":3048624466}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14562,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6837/1998956589","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6836","nonce":1998956589},{"type":"v1","addr":"172.21.15.159:6837","nonce":1998956589}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14619,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6839/793152328","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6838","nonce":793152328},{"type":"v1","addr":"172.21.15.159:6839","nonce":793152328}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14631,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.159:6841/536144142","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6840","nonce":536144142},{"type":"v1","addr":"172.21.15.159:6841","nonce":536144142}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24424,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6833/1022778932","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6832","nonce":1022778932},{"type":"v1","addr":"172.21.15.171:6833","nonce":1022778932}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24440,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6833/2969583384","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6832","nonce":2969583384},{"type":"v1","addr":"172.21.15.161:6833","nonce":2969583384}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24464,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6835/761787339","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6834","nonce":761787339},{"type":"v1","addr":"172.21.15.161:6835","nonce":761787339}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24472,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6835/1823903399","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6834","nonce":1823903399},{"type":"v1","addr":"172.21.15.171:6835","nonce":1823903399}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24484,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6837/2737069329","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6836","nonce":2737069329},{"type":"v1","addr":"172.21.15.171:6837","nonce":2737069329}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24491,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.161:6837/406989937","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6836","nonce":406989937},{"type":"v1","addr":"172.21.15.161:6837","nonce":406989937}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9},{"gid":24502,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.171:6839/1538859602","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6838","nonce":1538859602},{"type":"v1","addr":"172.21.15.171:6839","nonce":1538859602}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":10}],"filesystems":[{"mdsmap":{"epoch":17,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2023-03-27T10:22:31.255022+0000","modified":"2023-03-27T10:22:35.829205+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24539},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24539":{"gid":24539,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.161:6839/2491878460","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6838","nonce":2491878460},{"type":"v1","addr":"172.21.15.161:6839","nonce":2491878460}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":1},"id":1}]} 2023-03-27T10:22:37.264 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.265+0000 7fec5e681700 1 -- 172.21.15.159:0/320352801 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fec4009b400 msgr2=0x7fec4009d8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:37.264 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.265+0000 7fec5e681700 1 --2- 172.21.15.159:0/320352801 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fec4009b400 0x7fec4009d8b0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7fec4c009c60 tx=0x7fec4c009260 comp rx=0 tx=0).stop 2023-03-27T10:22:37.265 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.265+0000 7fec5e681700 1 -- 172.21.15.159:0/320352801 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fec580febd0 msgr2=0x7fec58197dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:37.265 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.265+0000 7fec5e681700 1 --2- 172.21.15.159:0/320352801 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fec580febd0 0x7fec58197dc0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7fec4800b530 tx=0x7fec4800b840 comp rx=0 tx=0).stop 2023-03-27T10:22:37.265 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.265+0000 7fec5e681700 1 -- 172.21.15.159:0/320352801 shutdown_connections 2023-03-27T10:22:37.265 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.265+0000 7fec5e681700 1 --2- 172.21.15.159:0/320352801 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fec4009b400 0x7fec4009d8b0 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.265 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.265+0000 7fec5e681700 1 --2- 172.21.15.159:0/320352801 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fec58100810 0x7fec5819f330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.265 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.265+0000 7fec5e681700 1 --2- 172.21.15.159:0/320352801 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fec580febd0 0x7fec58197dc0 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.265 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.265+0000 7fec5e681700 1 --2- 172.21.15.159:0/320352801 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fec580ffe60 0x7fec58198300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.266 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.265+0000 7fec5e681700 1 -- 172.21.15.159:0/320352801 >> 172.21.15.159:0/320352801 conn(0x7fec580fa210 msgr2=0x7fec580fc410 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:37.266 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.265+0000 7fec5e681700 1 -- 172.21.15.159:0/320352801 shutdown_connections 2023-03-27T10:22:37.266 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.265+0000 7fec5e681700 1 -- 172.21.15.159:0/320352801 wait complete. 2023-03-27T10:22:37.266 INFO:teuthology.orchestra.run.smithi159.stderr:dumped fsmap epoch 17 2023-03-27T10:22:37.277 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 17, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2023-03-27T10:22:31.255022+0000', 'modified': '2023-03-27T10:22:35.829205+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 24539}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24539': {'gid': 24539, 'name': 'b', 'rank': 0, 'incarnation': 14, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.161:6839/2491878460', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.161:6838', 'nonce': 2491878460}, {'type': 'v1', 'addr': '172.21.15.161:6839', 'nonce': 2491878460}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}}, 'data_pools': [3, 4], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 1} 2023-03-27T10:22:37.277 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2023-03-27T10:22:37.277 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2023-03-27T10:22:37.399 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.400+0000 7f9054a06700 1 -- 172.21.15.159:0/3774451962 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9050100810 msgr2=0x7f90501070b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:37.400 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.400+0000 7f9054a06700 1 --2- 172.21.15.159:0/3774451962 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9050100810 0x7f90501070b0 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f9048009f90 tx=0x7f904800b330 comp rx=0 tx=0).stop 2023-03-27T10:22:37.400 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.400+0000 7f9054a06700 1 -- 172.21.15.159:0/3774451962 shutdown_connections 2023-03-27T10:22:37.400 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.400+0000 7f9054a06700 1 --2- 172.21.15.159:0/3774451962 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f9050100810 0x7f90501070b0 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.400 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.400+0000 7f9054a06700 1 --2- 172.21.15.159:0/3774451962 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f90500febd0 0x7f90500fefe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.400 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.400+0000 7f9054a06700 1 --2- 172.21.15.159:0/3774451962 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f90500ffe60 0x7f90501002d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.400 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.400+0000 7f9054a06700 1 -- 172.21.15.159:0/3774451962 >> 172.21.15.159:0/3774451962 conn(0x7f90500fa210 msgr2=0x7f90500fc620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:37.400 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.400+0000 7f9054a06700 1 -- 172.21.15.159:0/3774451962 shutdown_connections 2023-03-27T10:22:37.400 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.400+0000 7f9054a06700 1 -- 172.21.15.159:0/3774451962 wait complete. 2023-03-27T10:22:37.402 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f9054a06700 1 Processor -- start 2023-03-27T10:22:37.402 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f9054a06700 1 -- start start 2023-03-27T10:22:37.402 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f9054a06700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f90500febd0 0x7f9050102c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.402 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f9054a06700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f90500ffe60 0x7f9050101240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.403 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f9054a06700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f9050100810 0x7f9050101780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.403 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f9054a06700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9050101f20 con 0x7f90500febd0 2023-03-27T10:22:37.403 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f9054a06700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9050102090 con 0x7f9050100810 2023-03-27T10:22:37.403 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f9054a06700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f905019fbb0 con 0x7f90500ffe60 2023-03-27T10:22:37.403 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f904e59c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f90500febd0 0x7f9050102c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.403 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f904e59c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f90500febd0 0x7f9050102c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45756/0 (socket says 172.21.15.159:45756) 2023-03-27T10:22:37.403 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f904e59c700 1 -- 172.21.15.159:0/597453146 learned_addr learned my addr 172.21.15.159:0/597453146 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:37.404 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f904dd9b700 1 --2- 172.21.15.159:0/597453146 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f90500ffe60 0x7f9050101240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.404 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f904ed9d700 1 --2- 172.21.15.159:0/597453146 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f9050100810 0x7f9050101780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.404 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f904e59c700 1 -- 172.21.15.159:0/597453146 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f90500ffe60 msgr2=0x7f9050101240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:37.404 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f904e59c700 1 --2- 172.21.15.159:0/597453146 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f90500ffe60 0x7f9050101240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.404 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f904e59c700 1 -- 172.21.15.159:0/597453146 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f9050100810 msgr2=0x7f9050101780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:37.404 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f904e59c700 1 --2- 172.21.15.159:0/597453146 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f9050100810 0x7f9050101780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.404 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f904e59c700 1 -- 172.21.15.159:0/597453146 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9048009c40 con 0x7f90500febd0 2023-03-27T10:22:37.404 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f904e59c700 1 --2- 172.21.15.159:0/597453146 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f90500febd0 0x7f9050102c00 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f904400d950 tx=0x7f904400dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:37.405 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f903f7fe700 1 -- 172.21.15.159:0/597453146 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9044009930 con 0x7f90500febd0 2023-03-27T10:22:37.405 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f903f7fe700 1 -- 172.21.15.159:0/597453146 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9044010460 con 0x7f90500febd0 2023-03-27T10:22:37.405 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.401+0000 7f903f7fe700 1 -- 172.21.15.159:0/597453146 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f904400f620 con 0x7f90500febd0 2023-03-27T10:22:37.405 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.402+0000 7f9054a06700 1 -- 172.21.15.159:0/597453146 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f905010a530 con 0x7f90500febd0 2023-03-27T10:22:37.405 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.402+0000 7f9054a06700 1 -- 172.21.15.159:0/597453146 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f905010a3a0 con 0x7f90500febd0 2023-03-27T10:22:37.410 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.404+0000 7f903f7fe700 1 -- 172.21.15.159:0/597453146 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f9044022070 con 0x7f90500febd0 2023-03-27T10:22:37.410 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.404+0000 7f903f7fe700 1 --2- 172.21.15.159:0/597453146 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f903809b3a0 0x7f903809d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.410 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.404+0000 7f903f7fe700 1 -- 172.21.15.159:0/597453146 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(89..89 src has 1..89) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f90440c4e40 con 0x7f90500febd0 2023-03-27T10:22:37.410 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.404+0000 7f904dd9b700 1 --2- 172.21.15.159:0/597453146 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f903809b3a0 0x7f903809d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.410 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.405+0000 7f904dd9b700 1 --2- 172.21.15.159:0/597453146 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f903809b3a0 0x7f903809d850 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f9040005fd0 tx=0x7f9040009310 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:37.410 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.405+0000 7f9060b88b80 1 Processor -- start 2023-03-27T10:22:37.411 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.405+0000 7f9060b88b80 1 -- start start 2023-03-27T10:22:37.411 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.405+0000 7f9060b88b80 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e0f110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.411 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.405+0000 7f9060b88b80 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e0f650 0x55cfa3e0ff70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.411 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.405+0000 7f9060b88b80 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e10970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.411 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.405+0000 7f9060b88b80 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cfa3cf9950 con 0x55cfa3e10500 2023-03-27T10:22:37.411 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.405+0000 7f9060b88b80 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cfa3d680d0 con 0x55cfa3e0f650 2023-03-27T10:22:37.411 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.405+0000 7f9060b88b80 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cfa3d61600 con 0x55cfa3e0ed00 2023-03-27T10:22:37.411 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.405+0000 7f904ed9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e10970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.412 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.405+0000 7f904ed9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e10970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45768/0 (socket says 172.21.15.159:45768) 2023-03-27T10:22:37.412 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.405+0000 7f904ed9d700 1 -- 172.21.15.159:0/3281588437 learned_addr learned my addr 172.21.15.159:0/3281588437 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:37.412 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.405+0000 7f904dd9b700 1 --2- 172.21.15.159:0/3281588437 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e0f650 0x55cfa3e0ff70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.412 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.405+0000 7f904e59c700 1 --2- 172.21.15.159:0/3281588437 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e0f110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.412 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.406+0000 7f904ed9d700 1 -- 172.21.15.159:0/3281588437 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 msgr2=0x55cfa3e0f110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-03-27T10:22:37.412 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.406+0000 7f904ed9d700 1 --2- 172.21.15.159:0/3281588437 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e0f110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.412 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.406+0000 7f904ed9d700 1 -- 172.21.15.159:0/3281588437 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e0f650 msgr2=0x55cfa3e0ff70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-03-27T10:22:37.412 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.406+0000 7f904ed9d700 1 --2- 172.21.15.159:0/3281588437 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e0f650 0x55cfa3e0ff70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.413 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.406+0000 7f904ed9d700 1 -- 172.21.15.159:0/3281588437 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cfa3dc4470 con 0x55cfa3e10500 2023-03-27T10:22:37.413 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.406+0000 7f904ed9d700 1 --2- 172.21.15.159:0/3281588437 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e10970 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f9048015000 tx=0x7f9048009bd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=546aa3e778b4349b server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:37.413 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.406+0000 7f903cff9700 1 -- 172.21.15.159:0/3281588437 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f904800e040 con 0x55cfa3e10500 2023-03-27T10:22:37.413 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.406+0000 7f903cff9700 1 -- 172.21.15.159:0/3281588437 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f90480074a0 con 0x55cfa3e10500 2023-03-27T10:22:37.413 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.406+0000 7f903cff9700 1 -- 172.21.15.159:0/3281588437 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f904800bd40 con 0x55cfa3e10500 2023-03-27T10:22:37.413 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 -- 172.21.15.159:0/3281588437 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 msgr2=0x55cfa3e10970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:37.413 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 --2- 172.21.15.159:0/3281588437 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e10970 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f9048015000 tx=0x7f9048009bd0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.414 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 -- 172.21.15.159:0/3281588437 shutdown_connections 2023-03-27T10:22:37.414 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 --2- 172.21.15.159:0/3281588437 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e0f110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.414 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 --2- 172.21.15.159:0/3281588437 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e10970 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.414 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 --2- 172.21.15.159:0/3281588437 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e0f650 0x55cfa3e0ff70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.414 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 -- 172.21.15.159:0/3281588437 >> 172.21.15.159:0/3281588437 conn(0x55cfa3e09810 msgr2=0x55cfa3e09c00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:37.414 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 -- 172.21.15.159:0/3281588437 shutdown_connections 2023-03-27T10:22:37.414 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 -- 172.21.15.159:0/3281588437 wait complete. 2023-03-27T10:22:37.414 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 -- start start 2023-03-27T10:22:37.415 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 Processor -- start 2023-03-27T10:22:37.415 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e18960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.415 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e0f650 0x55cfa3e18ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.415 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e1fee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.415 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cfa3be58d0 con 0x55cfa3e10500 2023-03-27T10:22:37.415 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cfa3e20420 con 0x55cfa3e0f650 2023-03-27T10:22:37.415 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cfa3e205b0 con 0x55cfa3e0ed00 2023-03-27T10:22:37.415 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.407+0000 7f9060b88b80 10 client.0 _ll_register_callbacks cb 0x55cfa3d074d0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-03-27T10:22:37.416 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f904ed9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e1fee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.416 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f904ed9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e1fee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45774/0 (socket says 172.21.15.159:45774) 2023-03-27T10:22:37.416 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f904ed9d700 1 -- 172.21.15.159:0/3104116741 learned_addr learned my addr 172.21.15.159:0/3104116741 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:37.416 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f904dd9b700 1 --2- 172.21.15.159:0/3104116741 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e0f650 0x55cfa3e18ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.416 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f904e59c700 1 --2- 172.21.15.159:0/3104116741 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e18960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.416 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f904ed9d700 1 -- 172.21.15.159:0/3104116741 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 msgr2=0x55cfa3e18960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-03-27T10:22:37.416 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f904ed9d700 1 --2- 172.21.15.159:0/3104116741 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e18960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.416 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f904ed9d700 1 -- 172.21.15.159:0/3104116741 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e0f650 msgr2=0x55cfa3e18ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-03-27T10:22:37.417 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f904ed9d700 1 --2- 172.21.15.159:0/3104116741 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e0f650 0x55cfa3e18ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.417 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f904ed9d700 1 -- 172.21.15.159:0/3104116741 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9050107920 con 0x55cfa3e10500 2023-03-27T10:22:37.417 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f904ed9d700 1 --2- 172.21.15.159:0/3104116741 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e1fee0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f9048009550 tx=0x7f9048008b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f6d9be88c774fdf2 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:37.417 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f9035ffb700 10 client.0 ms_handle_connect on v2:172.21.15.159:3300/0 2023-03-27T10:22:37.417 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f9035ffb700 1 -- 172.21.15.159:0/3104116741 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f904800e040 con 0x55cfa3e10500 2023-03-27T10:22:37.417 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f9035ffb700 1 -- 172.21.15.159:0/3104116741 <== mon.0 v2:172.21.15.159:3300/0 2 ==== osd_map(89..89 src has 1..89) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f904800bc60 con 0x55cfa3e10500 2023-03-27T10:22:37.417 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f9060b88b80 1 -- 172.21.15.159:0/3104116741 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55cfa3d23240 con 0x55cfa3e10500 2023-03-27T10:22:37.418 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f9035ffb700 1 -- 172.21.15.159:0/3104116741 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_get_version_reply(handle=1 version=17) v2 ==== 24+0+0 (secure 0 0 0) 0x7f904800be20 con 0x55cfa3e10500 2023-03-27T10:22:37.418 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f9060b88b80 10 client.14745 fetch_fsmap learned FSMap version 17 2023-03-27T10:22:37.418 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f9060b88b80 1 -- 172.21.15.159:0/3104116741 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,fsmap=17,monmap=4+}) v3 -- 0x55cfa3b8fd90 con 0x55cfa3e10500 2023-03-27T10:22:37.418 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.408+0000 7f9035ffb700 1 -- 172.21.15.159:0/3104116741 <== mon.0 v2:172.21.15.159:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55cfa3d23240 con 0x55cfa3e10500 2023-03-27T10:22:37.418 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.409+0000 7f9035ffb700 1 -- 172.21.15.159:0/3104116741 <== mon.0 v2:172.21.15.159:3300/0 5 ==== fsmap(e 17) v1 ==== 7474+0+0 (secure 0 0 0) 0x7f9048019030 con 0x55cfa3e10500 2023-03-27T10:22:37.418 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.409+0000 7f9060b88b80 10 client.14745 fetch_fsmap finished waiting for FSMap version 17 2023-03-27T10:22:37.418 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.409+0000 7f9060b88b80 10 client.14745 resolve_mds: resolved name 'b' to daemon mds.b 2023-03-27T10:22:37.418 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.409+0000 7f9060b88b80 20 client.14745 populate_metadata read hostname 'smithi159' 2023-03-27T10:22:37.419 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.409+0000 7f9060b88b80 1 --2- 172.21.15.159:0/3104116741 >> [v2:172.21.15.161:6838/2491878460,v1:172.21.15.161:6839/2491878460] conn(0x55cfa3e23a20 0x55cfa3e23090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.419 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.409+0000 7f9060b88b80 4 client.14745 mds_command: new command op to 24539 tid=0[{"prefix": "get_command_descriptions"}] 2023-03-27T10:22:37.419 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.409+0000 7f9060b88b80 1 -- 172.21.15.159:0/3104116741 --> [v2:172.21.15.161:6838/2491878460,v1:172.21.15.161:6839/2491878460] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55cfa3be5f10 con 0x55cfa3e23a20 2023-03-27T10:22:37.419 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.409+0000 7f904e59c700 1 --2- 172.21.15.159:0/3104116741 >> [v2:172.21.15.161:6838/2491878460,v1:172.21.15.161:6839/2491878460] conn(0x55cfa3e23a20 0x55cfa3e23090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.419 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.409+0000 7f9035ffb700 10 client.14745 ms_handle_connect on v2:172.21.15.161:6838/2491878460 2023-03-27T10:22:37.419 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.409+0000 7f904e59c700 1 --2- 172.21.15.159:0/3104116741 >> [v2:172.21.15.161:6838/2491878460,v1:172.21.15.161:6839/2491878460] conn(0x55cfa3e23a20 0x55cfa3e23090 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e2c78be2aa4711ba server_cookie=3379aa2ea341b50a in_seq=0 out_seq=0 2023-03-27T10:22:37.419 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.410+0000 7f9035ffb700 1 -- 172.21.15.159:0/3104116741 <== mds.0 v2:172.21.15.161:6838/2491878460 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+17527 (crc 0 0 0) 0x55cfa3be5f10 con 0x55cfa3e23a20 2023-03-27T10:22:37.419 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.410+0000 7f9035ffb700 10 client.14745 handle_command_reply: tid=0 2023-03-27T10:22:37.420 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.410+0000 7f9060b88b80 1 client.14745 shutdown 2023-03-27T10:22:37.420 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.410+0000 7f9060b88b80 1 -- 172.21.15.159:0/3104116741 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 msgr2=0x55cfa3e1fee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:37.420 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.410+0000 7f9060b88b80 1 --2- 172.21.15.159:0/3104116741 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e1fee0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f9048009550 tx=0x7f9048008b60 comp rx=0 tx=0).stop 2023-03-27T10:22:37.420 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.410+0000 7f9060b88b80 1 -- 172.21.15.159:0/3104116741 shutdown_connections 2023-03-27T10:22:37.420 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.410+0000 7f9060b88b80 1 --2- 172.21.15.159:0/3104116741 >> [v2:172.21.15.161:6838/2491878460,v1:172.21.15.161:6839/2491878460] conn(0x55cfa3e23a20 0x55cfa3e23090 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.420 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.410+0000 7f9060b88b80 1 --2- 172.21.15.159:0/3104116741 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e18960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.420 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.410+0000 7f9035ffb700 0 client.14745 ms_handle_reset on v2:172.21.15.161:6838/2491878460 2023-03-27T10:22:37.421 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.410+0000 7f9060b88b80 1 --2- 172.21.15.159:0/3104116741 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e1fee0 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.421 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.410+0000 7f9060b88b80 1 --2- 172.21.15.159:0/3104116741 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e0f650 0x55cfa3e18ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.421 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.410+0000 7f9060b88b80 1 -- 172.21.15.159:0/3104116741 >> 172.21.15.159:0/3104116741 conn(0x55cfa3e09810 msgr2=0x55cfa3e08ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:37.421 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.410+0000 7f9060b88b80 1 -- 172.21.15.159:0/3104116741 shutdown_connections 2023-03-27T10:22:37.421 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.410+0000 7f9060b88b80 1 -- 172.21.15.159:0/3104116741 wait complete. 2023-03-27T10:22:37.421 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.410+0000 7f9060b88b80 20 client.14745 trim_cache size 0 max 16384 2023-03-27T10:22:37.427 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.427+0000 7f9060b88b80 1 Processor -- start 2023-03-27T10:22:37.428 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.427+0000 7f9060b88b80 1 -- start start 2023-03-27T10:22:37.428 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f9060b88b80 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e23090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.428 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f9060b88b80 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e0f650 0x55cfa3e08ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.428 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f9060b88b80 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e10500 0x55cfa3e176b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.428 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f9060b88b80 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cfa3d62890 con 0x55cfa3e0f650 2023-03-27T10:22:37.428 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f9060b88b80 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cfa3d5d6f0 con 0x55cfa3e10500 2023-03-27T10:22:37.428 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f9060b88b80 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cfa3d59950 con 0x55cfa3e0ed00 2023-03-27T10:22:37.428 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f904dd9b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e0f650 0x55cfa3e08ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.429 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f904dd9b700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e0f650 0x55cfa3e08ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45780/0 (socket says 172.21.15.159:45780) 2023-03-27T10:22:37.429 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f904dd9b700 1 -- 172.21.15.159:0/3561061091 learned_addr learned my addr 172.21.15.159:0/3561061091 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:37.429 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f904e59c700 1 --2- 172.21.15.159:0/3561061091 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e23090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.429 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f904ed9d700 1 --2- 172.21.15.159:0/3561061091 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e10500 0x55cfa3e176b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.429 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f904dd9b700 1 -- 172.21.15.159:0/3561061091 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 msgr2=0x55cfa3e23090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-03-27T10:22:37.429 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f904dd9b700 1 --2- 172.21.15.159:0/3561061091 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e23090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.429 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f904dd9b700 1 -- 172.21.15.159:0/3561061091 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e10500 msgr2=0x55cfa3e176b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-03-27T10:22:37.430 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f904dd9b700 1 --2- 172.21.15.159:0/3561061091 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e10500 0x55cfa3e176b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.430 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f904dd9b700 1 -- 172.21.15.159:0/3561061091 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cfa3d28140 con 0x55cfa3e0f650 2023-03-27T10:22:37.430 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.428+0000 7f904dd9b700 1 --2- 172.21.15.159:0/3561061091 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e0f650 0x55cfa3e08ee0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f9040004200 tx=0x7f9040014000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a41d972114e64812 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:37.431 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.430+0000 7f903d7fa700 1 -- 172.21.15.159:0/3561061091 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9040024070 con 0x55cfa3e0f650 2023-03-27T10:22:37.431 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.431+0000 7f903d7fa700 1 -- 172.21.15.159:0/3561061091 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9040004510 con 0x55cfa3e0f650 2023-03-27T10:22:37.431 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.431+0000 7f903d7fa700 1 -- 172.21.15.159:0/3561061091 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90400283c0 con 0x55cfa3e0f650 2023-03-27T10:22:37.431 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.431+0000 7f9060b88b80 1 -- 172.21.15.159:0/3561061091 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e0f650 msgr2=0x55cfa3e08ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:37.431 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.431+0000 7f9060b88b80 1 --2- 172.21.15.159:0/3561061091 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e0f650 0x55cfa3e08ee0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f9040004200 tx=0x7f9040014000 comp rx=0 tx=0).stop 2023-03-27T10:22:37.431 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.431+0000 7f9060b88b80 1 -- 172.21.15.159:0/3561061091 shutdown_connections 2023-03-27T10:22:37.432 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.431+0000 7f9060b88b80 1 --2- 172.21.15.159:0/3561061091 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e23090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.432 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.431+0000 7f9060b88b80 1 --2- 172.21.15.159:0/3561061091 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e10500 0x55cfa3e176b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.432 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.431+0000 7f9060b88b80 1 --2- 172.21.15.159:0/3561061091 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e0f650 0x55cfa3e08ee0 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.432 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.431+0000 7f9060b88b80 1 -- 172.21.15.159:0/3561061091 >> 172.21.15.159:0/3561061091 conn(0x55cfa3e09810 msgr2=0x55cfa3e1fe00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:37.432 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.431+0000 7f9060b88b80 1 -- 172.21.15.159:0/3561061091 shutdown_connections 2023-03-27T10:22:37.432 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.431+0000 7f9060b88b80 1 -- 172.21.15.159:0/3561061091 wait complete. 2023-03-27T10:22:37.432 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f9060b88b80 1 -- start start 2023-03-27T10:22:37.432 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f9060b88b80 1 Processor -- start 2023-03-27T10:22:37.434 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f9060b88b80 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e279c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.435 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f9060b88b80 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e0f650 0x55cfa3e27f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.435 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f9060b88b80 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e2ac00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.435 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f9060b88b80 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cfa3e2b140 con 0x55cfa3e10500 2023-03-27T10:22:37.435 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f9060b88b80 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cfa3e2b2b0 con 0x55cfa3e0f650 2023-03-27T10:22:37.435 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f9060b88b80 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cfa3e2b470 con 0x55cfa3e0ed00 2023-03-27T10:22:37.435 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f9060b88b80 10 client.0 _ll_register_callbacks cb 0x55cfa3d074d0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-03-27T10:22:37.436 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f904ed9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e2ac00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.436 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f904e59c700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e279c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.436 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f904ed9d700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e2ac00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45788/0 (socket says 172.21.15.159:45788) 2023-03-27T10:22:37.436 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f904dd9b700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e0f650 0x55cfa3e27f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.436 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f904ed9d700 1 -- 172.21.15.159:0/1475690008 learned_addr learned my addr 172.21.15.159:0/1475690008 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:37.436 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f904e59c700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e279c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.171:3300/0 says I am v2:172.21.15.159:46322/0 (socket says 172.21.15.159:46322) 2023-03-27T10:22:37.436 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f904ed9d700 1 -- 172.21.15.159:0/1475690008 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 msgr2=0x55cfa3e279c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-03-27T10:22:37.436 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f904ed9d700 1 --2- 172.21.15.159:0/1475690008 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e279c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.437 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f904ed9d700 1 -- 172.21.15.159:0/1475690008 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e0f650 msgr2=0x55cfa3e27f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-03-27T10:22:37.437 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f904ed9d700 1 --2- 172.21.15.159:0/1475690008 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e0f650 0x55cfa3e27f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.437 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.432+0000 7f904ed9d700 1 -- 172.21.15.159:0/1475690008 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9040017070 con 0x55cfa3e10500 2023-03-27T10:22:37.437 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.433+0000 7f904ed9d700 1 --2- 172.21.15.159:0/1475690008 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e2ac00 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f9048009790 tx=0x7f904800bca0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=49c61222ae8a37be server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:37.437 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.433+0000 7f9035ffb700 10 client.0 ms_handle_connect on v2:172.21.15.159:3300/0 2023-03-27T10:22:37.437 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.434+0000 7f9035ffb700 1 -- 172.21.15.159:0/1475690008 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f904800e030 con 0x55cfa3e10500 2023-03-27T10:22:37.437 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.434+0000 7f9035ffb700 1 -- 172.21.15.159:0/1475690008 <== mon.0 v2:172.21.15.159:3300/0 2 ==== osd_map(89..89 src has 1..89) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f904801ce40 con 0x55cfa3e10500 2023-03-27T10:22:37.438 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.434+0000 7f9060b88b80 1 -- 172.21.15.159:0/1475690008 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55cfa3d1cde0 con 0x55cfa3e10500 2023-03-27T10:22:37.438 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.434+0000 7f9035ffb700 1 -- 172.21.15.159:0/1475690008 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_get_version_reply(handle=1 version=17) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9048008f60 con 0x55cfa3e10500 2023-03-27T10:22:37.438 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.434+0000 7f9060b88b80 10 client.14751 fetch_fsmap learned FSMap version 17 2023-03-27T10:22:37.438 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.434+0000 7f9060b88b80 1 -- 172.21.15.159:0/1475690008 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,fsmap=17,monmap=4+}) v3 -- 0x55cfa3bb6200 con 0x55cfa3e10500 2023-03-27T10:22:37.438 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.435+0000 7f9035ffb700 1 -- 172.21.15.159:0/1475690008 <== mon.0 v2:172.21.15.159:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55cfa3d1cde0 con 0x55cfa3e10500 2023-03-27T10:22:37.438 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.435+0000 7f9035ffb700 1 -- 172.21.15.159:0/1475690008 <== mon.0 v2:172.21.15.159:3300/0 5 ==== fsmap(e 17) v1 ==== 7474+0+0 (secure 0 0 0) 0x7f9048019070 con 0x55cfa3e10500 2023-03-27T10:22:37.438 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.435+0000 7f9060b88b80 10 client.14751 fetch_fsmap finished waiting for FSMap version 17 2023-03-27T10:22:37.438 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.435+0000 7f9060b88b80 10 client.14751 resolve_mds: resolved name 'b' to daemon mds.b 2023-03-27T10:22:37.439 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.435+0000 7f9060b88b80 20 client.14751 populate_metadata read hostname 'smithi159' 2023-03-27T10:22:37.439 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.435+0000 7f9060b88b80 1 --2- 172.21.15.159:0/1475690008 >> [v2:172.21.15.161:6838/2491878460,v1:172.21.15.161:6839/2491878460] conn(0x55cfa3e7a490 0x55cfa3e77280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:37.439 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.435+0000 7f9060b88b80 4 client.14751 mds_command: new command op to 24539 tid=0[{"prefix": "status"}] 2023-03-27T10:22:37.439 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.435+0000 7f9060b88b80 1 -- 172.21.15.159:0/1475690008 --> [v2:172.21.15.161:6838/2491878460,v1:172.21.15.161:6839/2491878460] -- command(tid 0: {"prefix": "status"}) v1 -- 0x55cfa3d02c80 con 0x55cfa3e7a490 2023-03-27T10:22:37.439 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.435+0000 7f904e59c700 1 --2- 172.21.15.159:0/1475690008 >> [v2:172.21.15.161:6838/2491878460,v1:172.21.15.161:6839/2491878460] conn(0x55cfa3e7a490 0x55cfa3e77280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:37.439 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9035ffb700 10 client.14751 ms_handle_connect on v2:172.21.15.161:6838/2491878460 2023-03-27T10:22:37.439 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f904e59c700 1 --2- 172.21.15.159:0/1475690008 >> [v2:172.21.15.161:6838/2491878460,v1:172.21.15.161:6839/2491878460] conn(0x55cfa3e7a490 0x55cfa3e77280 crc :-1 s=READY pgs=14 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a6753df09c98b0f4 server_cookie=3f52c959aaf80645 in_seq=0 out_seq=0 2023-03-27T10:22:37.440 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9035ffb700 1 -- 172.21.15.159:0/1475690008 <== mds.0 v2:172.21.15.161:6838/2491878460 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+326 (crc 0 0 0) 0x55cfa3d02c80 con 0x55cfa3e7a490 2023-03-27T10:22:37.440 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9035ffb700 10 client.14751 handle_command_reply: tid=0 2023-03-27T10:22:37.441 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9060b88b80 1 client.14751 shutdown 2023-03-27T10:22:37.441 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9060b88b80 1 -- 172.21.15.159:0/1475690008 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 msgr2=0x55cfa3e2ac00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:37.441 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9060b88b80 1 --2- 172.21.15.159:0/1475690008 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e2ac00 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f9048009790 tx=0x7f904800bca0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.441 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9060b88b80 1 -- 172.21.15.159:0/1475690008 shutdown_connections 2023-03-27T10:22:37.441 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9060b88b80 1 --2- 172.21.15.159:0/1475690008 >> [v2:172.21.15.161:6838/2491878460,v1:172.21.15.161:6839/2491878460] conn(0x55cfa3e7a490 0x55cfa3e77280 crc :-1 s=READY pgs=14 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.441 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9060b88b80 1 --2- 172.21.15.159:0/1475690008 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x55cfa3e0ed00 0x55cfa3e279c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.441 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9060b88b80 1 --2- 172.21.15.159:0/1475690008 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x55cfa3e10500 0x55cfa3e2ac00 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.442 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9035ffb700 0 client.14751 ms_handle_reset on v2:172.21.15.161:6838/2491878460 2023-03-27T10:22:37.442 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9060b88b80 1 --2- 172.21.15.159:0/1475690008 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x55cfa3e0f650 0x55cfa3e27f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.442 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9060b88b80 1 -- 172.21.15.159:0/1475690008 >> 172.21.15.159:0/1475690008 conn(0x55cfa3e09810 msgr2=0x55cfa3e23090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:37.442 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9060b88b80 1 -- 172.21.15.159:0/1475690008 shutdown_connections 2023-03-27T10:22:37.442 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9060b88b80 1 -- 172.21.15.159:0/1475690008 wait complete. 2023-03-27T10:22:37.442 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9060b88b80 20 client.14751 trim_cache size 0 max 16384 2023-03-27T10:22:37.443 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2023-03-27T10:22:37.443 INFO:teuthology.orchestra.run.smithi159.stdout: "cluster_fsid": "2c3cd140-cc88-11ed-9afe-001a4aab830c", 2023-03-27T10:22:37.443 INFO:teuthology.orchestra.run.smithi159.stdout: "whoami": 0, 2023-03-27T10:22:37.443 INFO:teuthology.orchestra.run.smithi159.stdout: "id": 24539, 2023-03-27T10:22:37.443 INFO:teuthology.orchestra.run.smithi159.stdout: "want_state": "up:active", 2023-03-27T10:22:37.443 INFO:teuthology.orchestra.run.smithi159.stdout: "state": "up:active", 2023-03-27T10:22:37.443 INFO:teuthology.orchestra.run.smithi159.stdout: "fs_name": "cephfs", 2023-03-27T10:22:37.444 INFO:teuthology.orchestra.run.smithi159.stdout: "rank_uptime": 6.1795061249999996, 2023-03-27T10:22:37.444 INFO:teuthology.orchestra.run.smithi159.stdout: "mdsmap_epoch": 17, 2023-03-27T10:22:37.444 INFO:teuthology.orchestra.run.smithi159.stdout: "osdmap_epoch": 89, 2023-03-27T10:22:37.444 INFO:teuthology.orchestra.run.smithi159.stdout: "osdmap_epoch_barrier": 88, 2023-03-27T10:22:37.444 INFO:teuthology.orchestra.run.smithi159.stdout: "uptime": 11.731150884 2023-03-27T10:22:37.444 INFO:teuthology.orchestra.run.smithi159.stdout:} 2023-03-27T10:22:37.445 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.436+0000 7f9054a06700 1 -- 172.21.15.159:0/597453146 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f903809b3a0 msgr2=0x7f903809d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:37.445 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.437+0000 7f9054a06700 1 --2- 172.21.15.159:0/597453146 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f903809b3a0 0x7f903809d850 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f9040005fd0 tx=0x7f9040009310 comp rx=0 tx=0).stop 2023-03-27T10:22:37.445 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.437+0000 7f9054a06700 1 -- 172.21.15.159:0/597453146 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f90500febd0 msgr2=0x7f9050102c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:37.445 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.437+0000 7f9054a06700 1 --2- 172.21.15.159:0/597453146 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f90500febd0 0x7f9050102c00 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f904400d950 tx=0x7f904400dc60 comp rx=0 tx=0).stop 2023-03-27T10:22:37.445 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.437+0000 7f9054a06700 1 -- 172.21.15.159:0/597453146 shutdown_connections 2023-03-27T10:22:37.446 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.437+0000 7f9054a06700 1 --2- 172.21.15.159:0/597453146 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f903809b3a0 0x7f903809d850 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.446 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.437+0000 7f9054a06700 1 --2- 172.21.15.159:0/597453146 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f9050100810 0x7f9050101780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.446 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.437+0000 7f9054a06700 1 --2- 172.21.15.159:0/597453146 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f90500febd0 0x7f9050102c00 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.446 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.437+0000 7f9054a06700 1 --2- 172.21.15.159:0/597453146 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f90500ffe60 0x7f9050101240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:37.446 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.437+0000 7f9054a06700 1 -- 172.21.15.159:0/597453146 >> 172.21.15.159:0/597453146 conn(0x7f90500fa210 msgr2=0x7f90500678e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:37.446 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.437+0000 7f9054a06700 1 -- 172.21.15.159:0/597453146 shutdown_connections 2023-03-27T10:22:37.446 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:37.437+0000 7f9054a06700 1 -- 172.21.15.159:0/597453146 wait complete. 2023-03-27T10:22:37.448 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2023-03-27T10:22:37.449 INFO:tasks.cephfs.mount:hostfs_mntpt = /home/ubuntu/cephtest/mnt.admin 2023-03-27T10:22:37.449 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b70fdf973c2ffa23c0d8aa6918519da05d922302 shell --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c -- ceph fs ls 2023-03-27T10:22:37.614 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:37 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/81782325' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:22:37.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:37 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3317798849' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:22:37.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:37 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/320352801' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:37.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:37 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/81782325' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:22:37.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:37 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3317798849' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:22:37.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:37 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/320352801' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:37.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:37 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/81782325' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:22:37.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:37 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3317798849' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:22:37.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:37 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/320352801' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-03-27T10:22:38.083 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/mon.a/config 2023-03-27T10:22:38.615 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:38 smithi159 ceph-mon[93039]: pgmap v264: 129 pgs: 1 creating+activating, 9 creating+peering, 110 active+clean, 9 unknown; 579 KiB data, 77 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s wr, 7 op/s 2023-03-27T10:22:38.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:38 smithi171 ceph-mon[94808]: pgmap v264: 129 pgs: 1 creating+activating, 9 creating+peering, 110 active+clean, 9 unknown; 579 KiB data, 77 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s wr, 7 op/s 2023-03-27T10:22:38.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:38 smithi161 ceph-mon[94333]: pgmap v264: 129 pgs: 1 creating+activating, 9 creating+peering, 110 active+clean, 9 unknown; 579 KiB data, 77 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s wr, 7 op/s 2023-03-27T10:22:39.391 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.391+0000 7fcb36d20700 1 -- 172.21.15.159:0/875291804 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fcb300ff8d0 msgr2=0x7fcb301038a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:39.391 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.391+0000 7fcb36d20700 1 --2- 172.21.15.159:0/875291804 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fcb300ff8d0 0x7fcb301038a0 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7fcb1c0099c0 tx=0x7fcb1c009cd0 comp rx=0 tx=0).stop 2023-03-27T10:22:39.391 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.391+0000 7fcb36d20700 1 -- 172.21.15.159:0/875291804 shutdown_connections 2023-03-27T10:22:39.391 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.391+0000 7fcb36d20700 1 --2- 172.21.15.159:0/875291804 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fcb30103fd0 0x7fcb3010bb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:39.392 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.391+0000 7fcb36d20700 1 --2- 172.21.15.159:0/875291804 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fcb300fef30 0x7fcb300ff300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:39.392 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.391+0000 7fcb36d20700 1 --2- 172.21.15.159:0/875291804 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fcb300ff8d0 0x7fcb301038a0 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:39.392 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.391+0000 7fcb36d20700 1 -- 172.21.15.159:0/875291804 >> 172.21.15.159:0/875291804 conn(0x7fcb300fa7f0 msgr2=0x7fcb300fcc00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:39.392 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.391+0000 7fcb36d20700 1 -- 172.21.15.159:0/875291804 shutdown_connections 2023-03-27T10:22:39.392 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.392+0000 7fcb36d20700 1 -- 172.21.15.159:0/875291804 wait complete. 2023-03-27T10:22:39.394 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.392+0000 7fcb36d20700 1 Processor -- start 2023-03-27T10:22:39.394 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.392+0000 7fcb36d20700 1 -- start start 2023-03-27T10:22:39.394 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.392+0000 7fcb36d20700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fcb300fef30 0x7fcb30196d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:39.394 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.392+0000 7fcb36d20700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fcb300ff8d0 0x7fcb301972a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:39.394 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.392+0000 7fcb36d20700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fcb30103fd0 0x7fcb301a1de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:39.395 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.392+0000 7fcb36d20700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcb30197b60 con 0x7fcb30103fd0 2023-03-27T10:22:39.395 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.392+0000 7fcb36d20700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcb301a2320 con 0x7fcb300fef30 2023-03-27T10:22:39.395 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.392+0000 7fcb36d20700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcb301a2460 con 0x7fcb300ff8d0 2023-03-27T10:22:39.395 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.392+0000 7fcb352bd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fcb30103fd0 0x7fcb301a1de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:39.395 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.392+0000 7fcb352bd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fcb30103fd0 0x7fcb301a1de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45800/0 (socket says 172.21.15.159:45800) 2023-03-27T10:22:39.395 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.392+0000 7fcb352bd700 1 -- 172.21.15.159:0/3050757797 learned_addr learned my addr 172.21.15.159:0/3050757797 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:39.395 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.393+0000 7fcb34abc700 1 --2- 172.21.15.159:0/3050757797 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fcb300fef30 0x7fcb30196d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:39.395 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.393+0000 7fcb352bd700 1 -- 172.21.15.159:0/3050757797 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fcb300ff8d0 msgr2=0x7fcb301972a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:39.396 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.393+0000 7fcb2ffff700 1 --2- 172.21.15.159:0/3050757797 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fcb300ff8d0 0x7fcb301972a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:39.396 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.393+0000 7fcb352bd700 1 --2- 172.21.15.159:0/3050757797 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fcb300ff8d0 0x7fcb301972a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:39.396 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.393+0000 7fcb352bd700 1 -- 172.21.15.159:0/3050757797 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fcb300fef30 msgr2=0x7fcb30196d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:39.396 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.393+0000 7fcb352bd700 1 --2- 172.21.15.159:0/3050757797 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fcb300fef30 0x7fcb30196d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:39.396 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.393+0000 7fcb352bd700 1 -- 172.21.15.159:0/3050757797 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcb1c0095f0 con 0x7fcb30103fd0 2023-03-27T10:22:39.396 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.393+0000 7fcb34abc700 1 --2- 172.21.15.159:0/3050757797 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fcb300fef30 0x7fcb30196d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-03-27T10:22:39.396 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.393+0000 7fcb352bd700 1 --2- 172.21.15.159:0/3050757797 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fcb30103fd0 0x7fcb301a1de0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7fcb28007ee0 tx=0x7fcb2800d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:39.396 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.393+0000 7fcb2dffb700 1 -- 172.21.15.159:0/3050757797 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcb28017070 con 0x7fcb30103fd0 2023-03-27T10:22:39.397 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.393+0000 7fcb2dffb700 1 -- 172.21.15.159:0/3050757797 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcb2800f040 con 0x7fcb30103fd0 2023-03-27T10:22:39.397 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.393+0000 7fcb36d20700 1 -- 172.21.15.159:0/3050757797 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcb3010ed00 con 0x7fcb30103fd0 2023-03-27T10:22:39.397 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.393+0000 7fcb2dffb700 1 -- 172.21.15.159:0/3050757797 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcb28005740 con 0x7fcb30103fd0 2023-03-27T10:22:39.397 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.393+0000 7fcb36d20700 1 -- 172.21.15.159:0/3050757797 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcb3010eb70 con 0x7fcb30103fd0 2023-03-27T10:22:39.397 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.394+0000 7fcb36d20700 1 -- 172.21.15.159:0/3050757797 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcb3010ee90 con 0x7fcb30103fd0 2023-03-27T10:22:39.398 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.395+0000 7fcb2dffb700 1 -- 172.21.15.159:0/3050757797 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7fcb280058a0 con 0x7fcb30103fd0 2023-03-27T10:22:39.398 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.396+0000 7fcb2dffb700 1 --2- 172.21.15.159:0/3050757797 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fcb2009b3a0 0x7fcb2009d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:39.398 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.396+0000 7fcb34abc700 1 --2- 172.21.15.159:0/3050757797 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fcb2009b3a0 0x7fcb2009d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:39.398 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.396+0000 7fcb2dffb700 1 -- 172.21.15.159:0/3050757797 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(89..89 src has 1..89) v4 ==== 8730+0+0 (secure 0 0 0) 0x7fcb280c4b30 con 0x7fcb30103fd0 2023-03-27T10:22:39.398 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.396+0000 7fcb34abc700 1 --2- 172.21.15.159:0/3050757797 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fcb2009b3a0 0x7fcb2009d850 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7fcb24005d90 tx=0x7fcb24005d00 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:39.399 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.400+0000 7fcb2dffb700 1 -- 172.21.15.159:0/3050757797 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7fcb28090200 con 0x7fcb30103fd0 2023-03-27T10:22:39.631 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.631+0000 7fcb36d20700 1 -- 172.21.15.159:0/3050757797 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7fcb301a2f10 con 0x7fcb30103fd0 2023-03-27T10:22:39.632 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.632+0000 7fcb2dffb700 1 -- 172.21.15.159:0/3050757797 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v17) v1 ==== 53+0+88 (secure 0 0 0) 0x7fcb28093820 con 0x7fcb30103fd0 2023-03-27T10:22:39.632 INFO:teuthology.orchestra.run.smithi159.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data cephfs_data_ec ] 2023-03-27T10:22:39.633 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.634+0000 7fcb36d20700 1 -- 172.21.15.159:0/3050757797 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fcb2009b3a0 msgr2=0x7fcb2009d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:39.633 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.634+0000 7fcb36d20700 1 --2- 172.21.15.159:0/3050757797 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fcb2009b3a0 0x7fcb2009d850 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7fcb24005d90 tx=0x7fcb24005d00 comp rx=0 tx=0).stop 2023-03-27T10:22:39.634 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.634+0000 7fcb36d20700 1 -- 172.21.15.159:0/3050757797 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fcb30103fd0 msgr2=0x7fcb301a1de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:39.634 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.634+0000 7fcb36d20700 1 --2- 172.21.15.159:0/3050757797 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fcb30103fd0 0x7fcb301a1de0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7fcb28007ee0 tx=0x7fcb2800d3b0 comp rx=0 tx=0).stop 2023-03-27T10:22:39.634 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.634+0000 7fcb36d20700 1 -- 172.21.15.159:0/3050757797 shutdown_connections 2023-03-27T10:22:39.634 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.634+0000 7fcb36d20700 1 --2- 172.21.15.159:0/3050757797 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7fcb2009b3a0 0x7fcb2009d850 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:39.634 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.634+0000 7fcb36d20700 1 --2- 172.21.15.159:0/3050757797 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7fcb30103fd0 0x7fcb301a1de0 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:39.634 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.634+0000 7fcb36d20700 1 --2- 172.21.15.159:0/3050757797 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7fcb300fef30 0x7fcb30196d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:39.634 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.634+0000 7fcb36d20700 1 --2- 172.21.15.159:0/3050757797 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7fcb300ff8d0 0x7fcb301972a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:39.635 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.634+0000 7fcb36d20700 1 -- 172.21.15.159:0/3050757797 >> 172.21.15.159:0/3050757797 conn(0x7fcb300fa7f0 msgr2=0x7fcb300fb280 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:39.635 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.634+0000 7fcb36d20700 1 -- 172.21.15.159:0/3050757797 shutdown_connections 2023-03-27T10:22:39.635 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:39.634+0000 7fcb36d20700 1 -- 172.21.15.159:0/3050757797 wait complete. 2023-03-27T10:22:40.039 INFO:tasks.cephfs.mount:Mounting Ceph FS cephfs; just confirmed its presence on cluster 2023-03-27T10:22:40.040 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2023-03-27T10:22:40.040 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi159.front.sepia.ceph.com 2023-03-27T10:22:40.040 INFO:tasks.cephfs.mount:self.client.name = client.admin 2023-03-27T10:22:40.040 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.admin 2023-03-27T10:22:40.040 INFO:tasks.cephfs.mount:self.cephfs_name = cephfs 2023-03-27T10:22:40.041 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2023-03-27T10:22:40.041 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2023-03-27T10:22:40.041 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/home/ubuntu/cephtest/mnt.admin' 2023-03-27T10:22:40.042 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-03-27T10:22:40.042 DEBUG:teuthology.orchestra.run.smithi159:> ip addr 2023-03-27T10:22:40.059 INFO:teuthology.orchestra.run.smithi159.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2023-03-27T10:22:40.060 INFO:teuthology.orchestra.run.smithi159.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2023-03-27T10:22:40.060 INFO:teuthology.orchestra.run.smithi159.stdout: inet 127.0.0.1/8 scope host lo 2023-03-27T10:22:40.060 INFO:teuthology.orchestra.run.smithi159.stdout: valid_lft forever preferred_lft forever 2023-03-27T10:22:40.060 INFO:teuthology.orchestra.run.smithi159.stdout: inet6 ::1/128 scope host 2023-03-27T10:22:40.060 INFO:teuthology.orchestra.run.smithi159.stdout: valid_lft forever preferred_lft forever 2023-03-27T10:22:40.060 INFO:teuthology.orchestra.run.smithi159.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2023-03-27T10:22:40.060 INFO:teuthology.orchestra.run.smithi159.stdout: link/ether 0c:c4:7a:d9:d8:c6 brd ff:ff:ff:ff:ff:ff 2023-03-27T10:22:40.060 INFO:teuthology.orchestra.run.smithi159.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2023-03-27T10:22:40.061 INFO:teuthology.orchestra.run.smithi159.stdout: link/ether 0c:c4:7a:d9:d8:c7 brd ff:ff:ff:ff:ff:ff 2023-03-27T10:22:40.061 INFO:teuthology.orchestra.run.smithi159.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2023-03-27T10:22:40.061 INFO:teuthology.orchestra.run.smithi159.stdout: link/ether 0c:c4:7a:88:72:4e brd ff:ff:ff:ff:ff:ff 2023-03-27T10:22:40.061 INFO:teuthology.orchestra.run.smithi159.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2023-03-27T10:22:40.061 INFO:teuthology.orchestra.run.smithi159.stdout: link/ether 0c:c4:7a:88:72:4f brd ff:ff:ff:ff:ff:ff 2023-03-27T10:22:40.061 INFO:teuthology.orchestra.run.smithi159.stdout: inet 172.21.15.159/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2023-03-27T10:22:40.061 INFO:teuthology.orchestra.run.smithi159.stdout: valid_lft 42051sec preferred_lft 42051sec 2023-03-27T10:22:40.062 INFO:teuthology.orchestra.run.smithi159.stdout: inet6 fe80::967c:9ee3:3b7f:68a0/64 scope link noprefixroute 2023-03-27T10:22:40.062 INFO:teuthology.orchestra.run.smithi159.stdout: valid_lft forever preferred_lft forever 2023-03-27T10:22:40.062 INFO:tasks.cephfs.mount:Setuping the 'ceph-brx' with 192.168.255.254/16 2023-03-27T10:22:40.063 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-03-27T10:22:40.063 DEBUG:teuthology.orchestra.run.smithi159:> (cd / && exec bash -c ' 2023-03-27T10:22:40.063 DEBUG:teuthology.orchestra.run.smithi159:> set -e 2023-03-27T10:22:40.063 DEBUG:teuthology.orchestra.run.smithi159:> sudo ip link add name ceph-brx type bridge 2023-03-27T10:22:40.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo ip addr flush dev ceph-brx 2023-03-27T10:22:40.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo ip link set ceph-brx up 2023-03-27T10:22:40.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo ip addr add 192.168.255.254/16 brd 192.168.255.255 dev ceph-brx 2023-03-27T10:22:40.064 DEBUG:teuthology.orchestra.run.smithi159:> ') 2023-03-27T10:22:40.263 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-03-27T10:22:40.263 DEBUG:teuthology.orchestra.run.smithi159:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2023-03-27T10:22:40.292 INFO:teuthology.orchestra.run.smithi159.stdout:1 2023-03-27T10:22:40.293 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-03-27T10:22:40.293 DEBUG:teuthology.orchestra.run.smithi159:> route 2023-03-27T10:22:40.360 INFO:teuthology.orchestra.run.smithi159.stdout:Kernel IP routing table 2023-03-27T10:22:40.361 INFO:teuthology.orchestra.run.smithi159.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2023-03-27T10:22:40.361 INFO:teuthology.orchestra.run.smithi159.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2023-03-27T10:22:40.361 INFO:teuthology.orchestra.run.smithi159.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2023-03-27T10:22:40.361 INFO:teuthology.orchestra.run.smithi159.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2023-03-27T10:22:40.362 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-03-27T10:22:40.362 DEBUG:teuthology.orchestra.run.smithi159:> (cd / && exec bash -c ' 2023-03-27T10:22:40.362 DEBUG:teuthology.orchestra.run.smithi159:> set -e 2023-03-27T10:22:40.362 DEBUG:teuthology.orchestra.run.smithi159:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2023-03-27T10:22:40.363 DEBUG:teuthology.orchestra.run.smithi159:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2023-03-27T10:22:40.363 DEBUG:teuthology.orchestra.run.smithi159:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2023-03-27T10:22:40.363 DEBUG:teuthology.orchestra.run.smithi159:> ') 2023-03-27T10:22:40.554 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:40 smithi159 ceph-mon[93039]: pgmap v265: 129 pgs: 1 creating+activating, 9 creating+peering, 112 active+clean, 7 unknown; 579 KiB data, 77 MiB used, 1.0 TiB / 1.0 TiB avail; 1.8 KiB/s wr, 5 op/s 2023-03-27T10:22:40.555 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:40 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3050757797' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2023-03-27T10:22:40.633 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-03-27T10:22:40.634 DEBUG:teuthology.orchestra.run.smithi159:> ip netns list 2023-03-27T10:22:40.651 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-03-27T10:22:40.651 DEBUG:teuthology.orchestra.run.smithi159:> ip netns list-id 2023-03-27T10:22:40.707 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-03-27T10:22:40.708 DEBUG:teuthology.orchestra.run.smithi159:> (cd / && exec bash -c ' 2023-03-27T10:22:40.708 DEBUG:teuthology.orchestra.run.smithi159:> set -e 2023-03-27T10:22:40.708 DEBUG:teuthology.orchestra.run.smithi159:> sudo ip netns add ceph-ns--home-ubuntu-cephtest-mnt.admin 2023-03-27T10:22:40.708 DEBUG:teuthology.orchestra.run.smithi159:> sudo ip netns set ceph-ns--home-ubuntu-cephtest-mnt.admin 0 2023-03-27T10:22:40.709 DEBUG:teuthology.orchestra.run.smithi159:> ') 2023-03-27T10:22:40.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:40 smithi171 ceph-mon[94808]: pgmap v265: 129 pgs: 1 creating+activating, 9 creating+peering, 112 active+clean, 7 unknown; 579 KiB data, 77 MiB used, 1.0 TiB / 1.0 TiB avail; 1.8 KiB/s wr, 5 op/s 2023-03-27T10:22:40.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:40 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3050757797' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2023-03-27T10:22:40.802 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--home-ubuntu-cephtest-mnt.admin' with 192.168.0.1/16 2023-03-27T10:22:40.802 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-03-27T10:22:40.802 DEBUG:teuthology.orchestra.run.smithi159:> (cd / && exec bash -c ' 2023-03-27T10:22:40.803 DEBUG:teuthology.orchestra.run.smithi159:> set -e 2023-03-27T10:22:40.803 DEBUG:teuthology.orchestra.run.smithi159:> sudo ip link add veth0 netns ceph-ns--home-ubuntu-cephtest-mnt.admin type veth peer name brx.0 2023-03-27T10:22:40.803 DEBUG:teuthology.orchestra.run.smithi159:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.admin ip addr add 192.168.0.1/16 brd 192.168.255.255 dev veth0 2023-03-27T10:22:40.803 DEBUG:teuthology.orchestra.run.smithi159:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.admin ip link set veth0 up 2023-03-27T10:22:40.804 DEBUG:teuthology.orchestra.run.smithi159:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.admin ip link set lo up 2023-03-27T10:22:40.804 DEBUG:teuthology.orchestra.run.smithi159:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.admin ip route add default via 192.168.255.254 2023-03-27T10:22:40.804 DEBUG:teuthology.orchestra.run.smithi159:> ') 2023-03-27T10:22:40.833 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:40 smithi161 ceph-mon[94333]: pgmap v265: 129 pgs: 1 creating+activating, 9 creating+peering, 112 active+clean, 7 unknown; 579 KiB data, 77 MiB used, 1.0 TiB / 1.0 TiB avail; 1.8 KiB/s wr, 5 op/s 2023-03-27T10:22:40.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:40 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3050757797' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2023-03-27T10:22:41.013 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-03-27T10:22:41.014 DEBUG:teuthology.orchestra.run.smithi159:> (cd / && exec bash -c ' 2023-03-27T10:22:41.014 DEBUG:teuthology.orchestra.run.smithi159:> set -e 2023-03-27T10:22:41.015 DEBUG:teuthology.orchestra.run.smithi159:> sudo ip link set brx.0 up 2023-03-27T10:22:41.015 DEBUG:teuthology.orchestra.run.smithi159:> sudo ip link set dev brx.0 master ceph-brx 2023-03-27T10:22:41.015 DEBUG:teuthology.orchestra.run.smithi159:> ') 2023-03-27T10:22:41.113 INFO:tasks.cephfs.fuse_mount:Client client.admin config is {} 2023-03-27T10:22:41.114 INFO:teuthology.orchestra.run:Running command with timeout 60 2023-03-27T10:22:41.114 DEBUG:teuthology.orchestra.run.smithi159:> mkdir -p -v /home/ubuntu/cephtest/mnt.admin 2023-03-27T10:22:41.130 INFO:teuthology.orchestra.run.smithi159.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.admin' 2023-03-27T10:22:41.131 INFO:teuthology.orchestra.run:Running command with timeout 60 2023-03-27T10:22:41.131 DEBUG:teuthology.orchestra.run.smithi159:> chmod 0000 /home/ubuntu/cephtest/mnt.admin 2023-03-27T10:22:41.187 DEBUG:teuthology.orchestra.run.smithi159:> sudo modprobe fuse 2023-03-27T10:22:41.253 INFO:teuthology.orchestra.run:Running command with timeout 30 2023-03-27T10:22:41.254 DEBUG:teuthology.orchestra.run.smithi159:> sudo mount -t fusectl /sys/fs/fuse/connections /sys/fs/fuse/connections 2023-03-27T10:22:41.320 INFO:teuthology.orchestra.run.smithi159.stderr:mount: /sys/fs/fuse/connections: /sys/fs/fuse/connections already mounted or mount point busy. 2023-03-27T10:22:41.321 DEBUG:teuthology.orchestra.run:got remote process result: 32 2023-03-27T10:22:41.322 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-03-27T10:22:41.322 DEBUG:teuthology.orchestra.run.smithi159:> ls /sys/fs/fuse/connections 2023-03-27T10:22:41.378 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [] 2023-03-27T10:22:41.378 DEBUG:teuthology.orchestra.run.smithi159:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.admin sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.admin --id admin --client_fs=cephfs) 2023-03-27T10:22:41.420 DEBUG:teuthology.orchestra.run.smithi159:> sudo modprobe fuse 2023-03-27T10:22:41.446 INFO:teuthology.orchestra.run:Running command with timeout 30 2023-03-27T10:22:41.446 DEBUG:teuthology.orchestra.run.smithi159:> sudo mount -t fusectl /sys/fs/fuse/connections /sys/fs/fuse/connections 2023-03-27T10:22:41.512 INFO:teuthology.orchestra.run.smithi159.stderr:mount: /sys/fs/fuse/connections: /sys/fs/fuse/connections already mounted or mount point busy. 2023-03-27T10:22:41.513 DEBUG:teuthology.orchestra.run:got remote process result: 32 2023-03-27T10:22:41.513 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-03-27T10:22:41.514 DEBUG:teuthology.orchestra.run.smithi159:> ls /sys/fs/fuse/connections 2023-03-27T10:22:41.525 INFO:tasks.cephfs.fuse_mount.ceph-fuse.admin.smithi159.stderr:2023-03-27T10:22:41.526+0000 7f3af528e5c0 -1 init, newargv = 0x5576adbac3a0 newargc=15 2023-03-27T10:22:41.525 INFO:tasks.cephfs.fuse_mount.ceph-fuse.admin.smithi159.stderr:ceph-fuse[130155]: starting ceph client 2023-03-27T10:22:41.531 INFO:tasks.cephfs.fuse_mount.ceph-fuse.admin.smithi159.stderr:ceph-fuse[130155]: starting fuse 2023-03-27T10:22:41.542 INFO:teuthology.orchestra.run.smithi159.stdout:124 2023-03-27T10:22:41.542 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [124] 2023-03-27T10:22:41.542 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits daemon-helper kill python3 -c ' 2023-03-27T10:22:41.543 DEBUG:teuthology.orchestra.run.smithi159:> import glob 2023-03-27T10:22:41.543 DEBUG:teuthology.orchestra.run.smithi159:> import re 2023-03-27T10:22:41.543 DEBUG:teuthology.orchestra.run.smithi159:> import os 2023-03-27T10:22:41.544 DEBUG:teuthology.orchestra.run.smithi159:> import subprocess 2023-03-27T10:22:41.544 DEBUG:teuthology.orchestra.run.smithi159:> 2023-03-27T10:22:41.544 DEBUG:teuthology.orchestra.run.smithi159:> def _find_admin_socket(client_name): 2023-03-27T10:22:41.544 DEBUG:teuthology.orchestra.run.smithi159:> asok_path = "/var/run/ceph/ceph-client.admin.*.asok" 2023-03-27T10:22:41.545 DEBUG:teuthology.orchestra.run.smithi159:> files = glob.glob(asok_path) 2023-03-27T10:22:41.545 DEBUG:teuthology.orchestra.run.smithi159:> mountpoint = "/home/ubuntu/cephtest/mnt.admin" 2023-03-27T10:22:41.545 DEBUG:teuthology.orchestra.run.smithi159:> 2023-03-27T10:22:41.545 DEBUG:teuthology.orchestra.run.smithi159:> # Given a non-glob path, it better be there 2023-03-27T10:22:41.546 DEBUG:teuthology.orchestra.run.smithi159:> if "*" not in asok_path: 2023-03-27T10:22:41.546 DEBUG:teuthology.orchestra.run.smithi159:> assert(len(files) == 1) 2023-03-27T10:22:41.546 DEBUG:teuthology.orchestra.run.smithi159:> return files[0] 2023-03-27T10:22:41.547 DEBUG:teuthology.orchestra.run.smithi159:> 2023-03-27T10:22:41.547 DEBUG:teuthology.orchestra.run.smithi159:> for f in files: 2023-03-27T10:22:41.547 DEBUG:teuthology.orchestra.run.smithi159:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2023-03-27T10:22:41.547 DEBUG:teuthology.orchestra.run.smithi159:> if os.path.exists("/proc/{0}".format(pid)): 2023-03-27T10:22:41.548 DEBUG:teuthology.orchestra.run.smithi159:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2023-03-27T10:22:41.548 DEBUG:teuthology.orchestra.run.smithi159:> contents = proc_f.read() 2023-03-27T10:22:41.548 DEBUG:teuthology.orchestra.run.smithi159:> if mountpoint in contents: 2023-03-27T10:22:41.548 DEBUG:teuthology.orchestra.run.smithi159:> return f 2023-03-27T10:22:41.549 DEBUG:teuthology.orchestra.run.smithi159:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2023-03-27T10:22:41.549 DEBUG:teuthology.orchestra.run.smithi159:> 2023-03-27T10:22:41.549 DEBUG:teuthology.orchestra.run.smithi159:> print(_find_admin_socket("client.admin")) 2023-03-27T10:22:41.549 DEBUG:teuthology.orchestra.run.smithi159:> ' 2023-03-27T10:22:41.665 INFO:teuthology.orchestra.run.smithi159.stdout:/var/run/ceph/ceph-client.admin.130155.asok 2023-03-27T10:22:41.848 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.admin.130155.asok 2023-03-27T10:22:41.849 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-03-27T10:22:41.849 DEBUG:teuthology.orchestra.run.smithi159:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.admin.130155.asok status 2023-03-27T10:22:42.006 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2023-03-27T10:22:42.006 INFO:teuthology.orchestra.run.smithi159.stdout: "metadata": { 2023-03-27T10:22:42.006 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph_sha1": "b70fdf973c2ffa23c0d8aa6918519da05d922302", 2023-03-27T10:22:42.006 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph_version": "ceph version 18.0.0-2533-gb70fdf97 (b70fdf973c2ffa23c0d8aa6918519da05d922302) reef (dev)", 2023-03-27T10:22:42.006 INFO:teuthology.orchestra.run.smithi159.stdout: "entity_id": "admin", 2023-03-27T10:22:42.007 INFO:teuthology.orchestra.run.smithi159.stdout: "hostname": "smithi159", 2023-03-27T10:22:42.007 INFO:teuthology.orchestra.run.smithi159.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.admin", 2023-03-27T10:22:42.007 INFO:teuthology.orchestra.run.smithi159.stdout: "pid": "130155", 2023-03-27T10:22:42.007 INFO:teuthology.orchestra.run.smithi159.stdout: "root": "/" 2023-03-27T10:22:42.007 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2023-03-27T10:22:42.007 INFO:teuthology.orchestra.run.smithi159.stdout: "dentry_count": 0, 2023-03-27T10:22:42.007 INFO:teuthology.orchestra.run.smithi159.stdout: "dentry_pinned_count": 0, 2023-03-27T10:22:42.008 INFO:teuthology.orchestra.run.smithi159.stdout: "id": 14763, 2023-03-27T10:22:42.008 INFO:teuthology.orchestra.run.smithi159.stdout: "inst": { 2023-03-27T10:22:42.008 INFO:teuthology.orchestra.run.smithi159.stdout: "name": { 2023-03-27T10:22:42.008 INFO:teuthology.orchestra.run.smithi159.stdout: "type": "client", 2023-03-27T10:22:42.008 INFO:teuthology.orchestra.run.smithi159.stdout: "num": 14763 2023-03-27T10:22:42.008 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2023-03-27T10:22:42.008 INFO:teuthology.orchestra.run.smithi159.stdout: "addr": { 2023-03-27T10:22:42.008 INFO:teuthology.orchestra.run.smithi159.stdout: "type": "v1", 2023-03-27T10:22:42.009 INFO:teuthology.orchestra.run.smithi159.stdout: "addr": "192.168.0.1:0", 2023-03-27T10:22:42.009 INFO:teuthology.orchestra.run.smithi159.stdout: "nonce": 1076028817 2023-03-27T10:22:42.009 INFO:teuthology.orchestra.run.smithi159.stdout: } 2023-03-27T10:22:42.009 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2023-03-27T10:22:42.009 INFO:teuthology.orchestra.run.smithi159.stdout: "addr": { 2023-03-27T10:22:42.009 INFO:teuthology.orchestra.run.smithi159.stdout: "type": "v1", 2023-03-27T10:22:42.009 INFO:teuthology.orchestra.run.smithi159.stdout: "addr": "192.168.0.1:0", 2023-03-27T10:22:42.010 INFO:teuthology.orchestra.run.smithi159.stdout: "nonce": 1076028817 2023-03-27T10:22:42.010 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2023-03-27T10:22:42.010 INFO:teuthology.orchestra.run.smithi159.stdout: "inst_str": "client.14763 192.168.0.1:0/1076028817", 2023-03-27T10:22:42.010 INFO:teuthology.orchestra.run.smithi159.stdout: "addr_str": "192.168.0.1:0/1076028817", 2023-03-27T10:22:42.010 INFO:teuthology.orchestra.run.smithi159.stdout: "inode_count": 1, 2023-03-27T10:22:42.010 INFO:teuthology.orchestra.run.smithi159.stdout: "mds_epoch": 17, 2023-03-27T10:22:42.010 INFO:teuthology.orchestra.run.smithi159.stdout: "osd_epoch": 89, 2023-03-27T10:22:42.010 INFO:teuthology.orchestra.run.smithi159.stdout: "osd_epoch_barrier": 0, 2023-03-27T10:22:42.011 INFO:teuthology.orchestra.run.smithi159.stdout: "blocklisted": false, 2023-03-27T10:22:42.011 INFO:teuthology.orchestra.run.smithi159.stdout: "fs_name": "cephfs" 2023-03-27T10:22:42.011 INFO:teuthology.orchestra.run.smithi159.stdout:} 2023-03-27T10:22:42.016 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-03-27T10:22:42.016 DEBUG:teuthology.orchestra.run.smithi159:> stat --file-system '--printf=%T 2023-03-27T10:22:42.016 DEBUG:teuthology.orchestra.run.smithi159:> ' -- /home/ubuntu/cephtest/mnt.admin 2023-03-27T10:22:42.033 INFO:teuthology.orchestra.run.smithi159.stdout:fuseblk 2023-03-27T10:22:42.034 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.admin 2023-03-27T10:22:42.034 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-03-27T10:22:42.034 DEBUG:teuthology.orchestra.run.smithi159:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.admin 2023-03-27T10:22:42.100 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-03-27T10:22:42.100 DEBUG:teuthology.orchestra.run.smithi159:> (cd /home/ubuntu/cephtest/mnt.admin && exec bash -c 'setfattr -n ceph.dir.layout.pool -v cephfs_data_ec . && getfattr -n ceph.dir.layout .') 2023-03-27T10:22:42.206 INFO:teuthology.orchestra.run.smithi159.stdout:# file: . 2023-03-27T10:22:42.206 INFO:teuthology.orchestra.run.smithi159.stdout:ceph.dir.layout="stripe_unit=4194304 stripe_count=1 object_size=4194304 pool=cephfs_data_ec" 2023-03-27T10:22:42.206 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:42.207 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2023-03-27T10:22:42.207 DEBUG:teuthology.orchestra.run.smithi159:> dd if=/proc/self/mounts of=/dev/stdout 2023-03-27T10:22:42.224 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-03-27T10:22:42.384 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.385+0000 7f829bc74700 1 -- 172.21.15.159:0/1503034187 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82940febd0 msgr2=0x7f82940fefe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:42.385 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.385+0000 7f829bc74700 1 --2- 172.21.15.159:0/1503034187 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82940febd0 0x7f82940fefe0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f8288009990 tx=0x7f8288009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.385 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.385+0000 7f829bc74700 1 -- 172.21.15.159:0/1503034187 shutdown_connections 2023-03-27T10:22:42.385 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.385+0000 7f829bc74700 1 --2- 172.21.15.159:0/1503034187 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f8294100810 0x7f82941070b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.385 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.385+0000 7f829bc74700 1 --2- 172.21.15.159:0/1503034187 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82940febd0 0x7f82940fefe0 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.386 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.385+0000 7f829bc74700 1 --2- 172.21.15.159:0/1503034187 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f82940ffe60 0x7f82941002d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.386 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.385+0000 7f829bc74700 1 -- 172.21.15.159:0/1503034187 >> 172.21.15.159:0/1503034187 conn(0x7f82940fa210 msgr2=0x7f82940fc620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:42.386 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.385+0000 7f829bc74700 1 -- 172.21.15.159:0/1503034187 shutdown_connections 2023-03-27T10:22:42.386 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.385+0000 7f829bc74700 1 -- 172.21.15.159:0/1503034187 wait complete. 2023-03-27T10:22:42.386 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.385+0000 7f829bc74700 1 Processor -- start 2023-03-27T10:22:42.386 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829bc74700 1 -- start start 2023-03-27T10:22:42.388 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829bc74700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f82940febd0 0x7f8294197c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:42.388 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829bc74700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82940ffe60 0x7f82941981c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:42.388 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829bc74700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f8294100810 0x7f829419f220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:42.388 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829bc74700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f829419f7f0 con 0x7f82940ffe60 2023-03-27T10:22:42.388 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829bc74700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f829419f960 con 0x7f82940febd0 2023-03-27T10:22:42.388 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829bc74700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f829419fb20 con 0x7f8294100810 2023-03-27T10:22:42.389 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829920f700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82940ffe60 0x7f82941981c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:42.389 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829920f700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82940ffe60 0x7f82941981c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45824/0 (socket says 172.21.15.159:45824) 2023-03-27T10:22:42.389 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829920f700 1 -- 172.21.15.159:0/1298482205 learned_addr learned my addr 172.21.15.159:0/1298482205 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:42.389 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829a211700 1 --2- 172.21.15.159:0/1298482205 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f8294100810 0x7f829419f220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:42.389 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f8299a10700 1 --2- 172.21.15.159:0/1298482205 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f82940febd0 0x7f8294197c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:42.389 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829920f700 1 -- 172.21.15.159:0/1298482205 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f8294100810 msgr2=0x7f829419f220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:42.389 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829920f700 1 --2- 172.21.15.159:0/1298482205 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f8294100810 0x7f829419f220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.390 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829920f700 1 -- 172.21.15.159:0/1298482205 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f82940febd0 msgr2=0x7f8294197c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:42.390 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829920f700 1 --2- 172.21.15.159:0/1298482205 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f82940febd0 0x7f8294197c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.390 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829920f700 1 -- 172.21.15.159:0/1298482205 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f82880095f0 con 0x7f82940ffe60 2023-03-27T10:22:42.390 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829920f700 1 --2- 172.21.15.159:0/1298482205 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82940ffe60 0x7f82941981c0 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f829000c920 tx=0x7f829000cc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:42.390 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f8286ffd700 1 -- 172.21.15.159:0/1298482205 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8290007b00 con 0x7f82940ffe60 2023-03-27T10:22:42.390 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f8286ffd700 1 -- 172.21.15.159:0/1298482205 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f829000ce70 con 0x7f82940ffe60 2023-03-27T10:22:42.390 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829bc74700 1 -- 172.21.15.159:0/1298482205 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f829410a530 con 0x7f82940ffe60 2023-03-27T10:22:42.390 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f8286ffd700 1 -- 172.21.15.159:0/1298482205 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f82900054e0 con 0x7f82940ffe60 2023-03-27T10:22:42.391 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.386+0000 7f829bc74700 1 -- 172.21.15.159:0/1298482205 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f829410a3a0 con 0x7f82940ffe60 2023-03-27T10:22:42.391 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.387+0000 7f829bc74700 1 -- 172.21.15.159:0/1298482205 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f82941a01e0 con 0x7f82940ffe60 2023-03-27T10:22:42.391 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.388+0000 7f8286ffd700 1 -- 172.21.15.159:0/1298482205 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f829001f070 con 0x7f82940ffe60 2023-03-27T10:22:42.391 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.389+0000 7f8286ffd700 1 --2- 172.21.15.159:0/1298482205 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f828009b3a0 0x7f828009d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:42.392 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.389+0000 7f8299a10700 1 --2- 172.21.15.159:0/1298482205 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f828009b3a0 0x7f828009d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:42.392 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.389+0000 7f8286ffd700 1 -- 172.21.15.159:0/1298482205 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(89..89 src has 1..89) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f82900c4fa0 con 0x7f82940ffe60 2023-03-27T10:22:42.392 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.389+0000 7f8299a10700 1 --2- 172.21.15.159:0/1298482205 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f828009b3a0 0x7f828009d850 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f828800b5c0 tx=0x7f8288005fd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:42.392 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.392+0000 7f8286ffd700 1 -- 172.21.15.159:0/1298482205 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f82900906d0 con 0x7f82940ffe60 2023-03-27T10:22:42.577 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.578+0000 7f829bc74700 1 -- 172.21.15.159:0/1298482205 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f82941a0880 con 0x7f82940ffe60 2023-03-27T10:22:42.578 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.579+0000 7f8286ffd700 1 -- 172.21.15.159:0/1298482205 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) v1 ==== 74+0+23993 (secure 0 0 0) 0x7f829001d070 con 0x7f82940ffe60 2023-03-27T10:22:42.579 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-03-27T10:22:42.579 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":89,"fsid":"2c3cd140-cc88-11ed-9afe-001a4aab830c","created":"2023-03-27T10:15:32.617230+0000","modified":"2023-03-27T10:22:35.828925+0000","last_up_change":"2023-03-27T10:21:49.442480+0000","last_in_change":"2023-03-27T10:21:32.265278+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-03-27T10:18:35.277335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-03-27T10:22:29.479987+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-03-27T10:22:30.424727+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-03-27T10:22:33.650889+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"89","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"f47f99b9-47f9-4d94-83d9-33b1278be17e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6803","nonce":87273908}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6805","nonce":87273908}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6809","nonce":87273908}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":87273908},{"type":"v1","addr":"172.21.15.159:6807","nonce":87273908}]},"public_addr":"172.21.15.159:6803/87273908","cluster_addr":"172.21.15.159:6805/87273908","heartbeat_back_addr":"172.21.15.159:6809/87273908","heartbeat_front_addr":"172.21.15.159:6807/87273908","state":["exists","up"]},{"osd":1,"uuid":"97f66da9-22b4-4eb5-9c77-9d37cf0fca82","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6800","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6801","nonce":2094166586}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6802","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6803","nonce":2094166586}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6806","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6807","nonce":2094166586}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6804","nonce":2094166586},{"type":"v1","addr":"172.21.15.161:6805","nonce":2094166586}]},"public_addr":"172.21.15.161:6801/2094166586","cluster_addr":"172.21.15.161:6803/2094166586","heartbeat_back_addr":"172.21.15.161:6807/2094166586","heartbeat_front_addr":"172.21.15.161:6805/2094166586","state":["exists","up"]},{"osd":2,"uuid":"ba9f5f82-3aef-42f7-a9e6-0e07aa7d8df1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6801","nonce":2724827688}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6803","nonce":2724827688}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6807","nonce":2724827688}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":2724827688},{"type":"v1","addr":"172.21.15.171:6805","nonce":2724827688}]},"public_addr":"172.21.15.171:6801/2724827688","cluster_addr":"172.21.15.171:6803/2724827688","heartbeat_back_addr":"172.21.15.171:6807/2724827688","heartbeat_front_addr":"172.21.15.171:6805/2724827688","state":["exists","up"]},{"osd":3,"uuid":"4ef4b519-fdef-4d38-b387-bc14ea381bd3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6811","nonce":1992276871}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6813","nonce":1992276871}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6817","nonce":1992276871}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":1992276871},{"type":"v1","addr":"172.21.15.159:6815","nonce":1992276871}]},"public_addr":"172.21.15.159:6811/1992276871","cluster_addr":"172.21.15.159:6813/1992276871","heartbeat_back_addr":"172.21.15.159:6817/1992276871","heartbeat_front_addr":"172.21.15.159:6815/1992276871","state":["exists","up"]},{"osd":4,"uuid":"982675c1-dd87-46b4-97ef-a5dde1b1d777","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6808","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6809","nonce":3978602555}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6810","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6811","nonce":3978602555}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6814","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6815","nonce":3978602555}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6812","nonce":3978602555},{"type":"v1","addr":"172.21.15.161:6813","nonce":3978602555}]},"public_addr":"172.21.15.161:6809/3978602555","cluster_addr":"172.21.15.161:6811/3978602555","heartbeat_back_addr":"172.21.15.161:6815/3978602555","heartbeat_front_addr":"172.21.15.161:6813/3978602555","state":["exists","up"]},{"osd":5,"uuid":"b4de4a85-4f91-4194-a1ce-db12f1898c4f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6809","nonce":569611891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6811","nonce":569611891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6815","nonce":569611891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":569611891},{"type":"v1","addr":"172.21.15.171:6813","nonce":569611891}]},"public_addr":"172.21.15.171:6809/569611891","cluster_addr":"172.21.15.171:6811/569611891","heartbeat_back_addr":"172.21.15.171:6815/569611891","heartbeat_front_addr":"172.21.15.171:6813/569611891","state":["exists","up"]},{"osd":6,"uuid":"dff7d5f4-655f-4baa-9084-03591a4b3bb9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6819","nonce":1176978350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6821","nonce":1176978350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6825","nonce":1176978350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":1176978350},{"type":"v1","addr":"172.21.15.159:6823","nonce":1176978350}]},"public_addr":"172.21.15.159:6819/1176978350","cluster_addr":"172.21.15.159:6821/1176978350","heartbeat_back_addr":"172.21.15.159:6825/1176978350","heartbeat_front_addr":"172.21.15.159:6823/1176978350","state":["exists","up"]},{"osd":7,"uuid":"30f026d9-6963-4468-95e8-78c7ed2e89f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6816","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6817","nonce":2098174471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6818","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6819","nonce":2098174471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6822","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6823","nonce":2098174471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6820","nonce":2098174471},{"type":"v1","addr":"172.21.15.161:6821","nonce":2098174471}]},"public_addr":"172.21.15.161:6817/2098174471","cluster_addr":"172.21.15.161:6819/2098174471","heartbeat_back_addr":"172.21.15.161:6823/2098174471","heartbeat_front_addr":"172.21.15.161:6821/2098174471","state":["exists","up"]},{"osd":8,"uuid":"9f69fbcc-32ff-4d4b-b562-ee0a12745cc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6817","nonce":1033586906}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6819","nonce":1033586906}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6823","nonce":1033586906}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":1033586906},{"type":"v1","addr":"172.21.15.171:6821","nonce":1033586906}]},"public_addr":"172.21.15.171:6817/1033586906","cluster_addr":"172.21.15.171:6819/1033586906","heartbeat_back_addr":"172.21.15.171:6823/1033586906","heartbeat_front_addr":"172.21.15.171:6821/1033586906","state":["exists","up"]},{"osd":9,"uuid":"d7fb9eed-901b-4943-9529-6edcba2385e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6827","nonce":3285179829}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6829","nonce":3285179829}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6832","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6833","nonce":3285179829}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":3285179829},{"type":"v1","addr":"172.21.15.159:6831","nonce":3285179829}]},"public_addr":"172.21.15.159:6827/3285179829","cluster_addr":"172.21.15.159:6829/3285179829","heartbeat_back_addr":"172.21.15.159:6833/3285179829","heartbeat_front_addr":"172.21.15.159:6831/3285179829","state":["exists","up"]},{"osd":10,"uuid":"4871ca03-30ae-4e87-942c-9441dd31fb40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":83,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6824","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6825","nonce":946677932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6826","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6827","nonce":946677932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6830","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6831","nonce":946677932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.161:6828","nonce":946677932},{"type":"v1","addr":"172.21.15.161:6829","nonce":946677932}]},"public_addr":"172.21.15.161:6825/946677932","cluster_addr":"172.21.15.161:6827/946677932","heartbeat_back_addr":"172.21.15.161:6831/946677932","heartbeat_front_addr":"172.21.15.161:6829/946677932","state":["exists","up"]},{"osd":11,"uuid":"1fb37d9a-ed52-4c0d-a082-65e17d242106","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":80,"up_thru":87,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6825","nonce":596725174}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6827","nonce":596725174}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6831","nonce":596725174}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":596725174},{"type":"v1","addr":"172.21.15.171:6829","nonce":596725174}]},"public_addr":"172.21.15.171:6825/596725174","cluster_addr":"172.21.15.171:6827/596725174","heartbeat_back_addr":"172.21.15.171:6831/596725174","heartbeat_front_addr":"172.21.15.171:6829/596725174","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:17:49.128592+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:10.944593+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:32.989432+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:18:54.675294+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:16.543485+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:38.390326+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:19:59.566202+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:20:21.348420+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:20:43.383576+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:04.710992+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:26.286573+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-03-27T10:21:48.277317+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.159:0/653469093":"2023-03-28T10:16:26.900382+0000","172.21.15.159:6800/3561797228":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/3045586257":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/2859804517":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6801/3561797228":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/3487813906":"2023-03-28T10:16:26.900382+0000","172.21.15.159:0/1306320900":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6800/1729835725":"2023-03-28T10:16:04.737100+0000","172.21.15.159:6801/1729835725":"2023-03-28T10:16:04.737100+0000","172.21.15.159:0/72206061":"2023-03-28T10:16:04.737100+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-03-27T10:22:42.581 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.581+0000 7f829bc74700 1 -- 172.21.15.159:0/1298482205 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f828009b3a0 msgr2=0x7f828009d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:42.581 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.581+0000 7f829bc74700 1 --2- 172.21.15.159:0/1298482205 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f828009b3a0 0x7f828009d850 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f828800b5c0 tx=0x7f8288005fd0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.581 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.581+0000 7f829bc74700 1 -- 172.21.15.159:0/1298482205 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82940ffe60 msgr2=0x7f82941981c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:42.581 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.581+0000 7f829bc74700 1 --2- 172.21.15.159:0/1298482205 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82940ffe60 0x7f82941981c0 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f829000c920 tx=0x7f829000cc30 comp rx=0 tx=0).stop 2023-03-27T10:22:42.581 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.581+0000 7f829bc74700 1 -- 172.21.15.159:0/1298482205 shutdown_connections 2023-03-27T10:22:42.582 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.581+0000 7f829bc74700 1 --2- 172.21.15.159:0/1298482205 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f828009b3a0 0x7f828009d850 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.582 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.581+0000 7f829bc74700 1 --2- 172.21.15.159:0/1298482205 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f8294100810 0x7f829419f220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.582 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.581+0000 7f829bc74700 1 --2- 172.21.15.159:0/1298482205 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f82940febd0 0x7f8294197c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.582 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.581+0000 7f829bc74700 1 --2- 172.21.15.159:0/1298482205 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f82940ffe60 0x7f82941981c0 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.582 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.581+0000 7f829bc74700 1 -- 172.21.15.159:0/1298482205 >> 172.21.15.159:0/1298482205 conn(0x7f82940fa210 msgr2=0x7f8294067900 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:42.582 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.582+0000 7f829bc74700 1 -- 172.21.15.159:0/1298482205 shutdown_connections 2023-03-27T10:22:42.582 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.582+0000 7f829bc74700 1 -- 172.21.15.159:0/1298482205 wait complete. 2023-03-27T10:22:42.593 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-03-27T10:18:35.277335+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2023-03-27T10:22:29.479987+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2023-03-27T10:22:30.424727+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '85', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2023-03-27T10:22:33.650889+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '89', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2023-03-27T10:22:42.594 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2023-03-27T10:22:42.718 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.719+0000 7f1f3d15a700 1 -- 172.21.15.159:0/4032139952 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1f380fe360 msgr2=0x7f1f380fe770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:42.718 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.719+0000 7f1f3d15a700 1 --2- 172.21.15.159:0/4032139952 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1f380fe360 0x7f1f380fe770 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f1f20009990 tx=0x7f1f20009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.719+0000 7f1f3d15a700 1 -- 172.21.15.159:0/4032139952 shutdown_connections 2023-03-27T10:22:42.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.719+0000 7f1f3d15a700 1 --2- 172.21.15.159:0/4032139952 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1f380ffee0 0x7f1f381067a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.719+0000 7f1f3d15a700 1 --2- 172.21.15.159:0/4032139952 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1f380fe360 0x7f1f380fe770 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.719+0000 7f1f3d15a700 1 --2- 172.21.15.159:0/4032139952 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1f380ff550 0x7f1f380ff9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.719+0000 7f1f3d15a700 1 -- 172.21.15.159:0/4032139952 >> 172.21.15.159:0/4032139952 conn(0x7f1f380f99a0 msgr2=0x7f1f380fbdb0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:42.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.719+0000 7f1f3d15a700 1 -- 172.21.15.159:0/4032139952 shutdown_connections 2023-03-27T10:22:42.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.719+0000 7f1f3d15a700 1 -- 172.21.15.159:0/4032139952 wait complete. 2023-03-27T10:22:42.721 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.719+0000 7f1f3d15a700 1 Processor -- start 2023-03-27T10:22:42.721 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3d15a700 1 -- start start 2023-03-27T10:22:42.721 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3d15a700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1f380fe360 0x7f1f381959f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:42.722 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3d15a700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1f380ff550 0x7f1f38195f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:42.722 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3d15a700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1f380ffee0 0x7f1f3819cfa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:42.722 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3d15a700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f3819d570 con 0x7f1f380ff550 2023-03-27T10:22:42.722 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3d15a700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f3819d6b0 con 0x7f1f380ffee0 2023-03-27T10:22:42.722 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3d15a700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1f3819d840 con 0x7f1f380fe360 2023-03-27T10:22:42.722 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3659c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1f380ff550 0x7f1f38195f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:42.722 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3659c700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1f380ff550 0x7f1f38195f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45838/0 (socket says 172.21.15.159:45838) 2023-03-27T10:22:42.723 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3659c700 1 -- 172.21.15.159:0/3315724478 learned_addr learned my addr 172.21.15.159:0/3315724478 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:42.723 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f36d9d700 1 --2- 172.21.15.159:0/3315724478 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1f380fe360 0x7f1f381959f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:42.723 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3759e700 1 --2- 172.21.15.159:0/3315724478 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1f380ffee0 0x7f1f3819cfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:42.723 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3659c700 1 -- 172.21.15.159:0/3315724478 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1f380fe360 msgr2=0x7f1f381959f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:42.723 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3659c700 1 --2- 172.21.15.159:0/3315724478 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1f380fe360 0x7f1f381959f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.723 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3659c700 1 -- 172.21.15.159:0/3315724478 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1f380ffee0 msgr2=0x7f1f3819cfa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:42.723 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3659c700 1 --2- 172.21.15.159:0/3315724478 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1f380ffee0 0x7f1f3819cfa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.723 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3659c700 1 -- 172.21.15.159:0/3315724478 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1f200095f0 con 0x7f1f380ff550 2023-03-27T10:22:42.724 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f3759e700 1 --2- 172.21.15.159:0/3315724478 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1f380ffee0 0x7f1f3819cfa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:22:42.724 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.720+0000 7f1f36d9d700 1 --2- 172.21.15.159:0/3315724478 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1f380fe360 0x7f1f381959f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:22:42.724 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.721+0000 7f1f3659c700 1 --2- 172.21.15.159:0/3315724478 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1f380ff550 0x7f1f38195f30 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f1f2800c9b0 tx=0x7f1f2800cd70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:42.724 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.721+0000 7f1f2ffff700 1 -- 172.21.15.159:0/3315724478 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f28007b00 con 0x7f1f380ff550 2023-03-27T10:22:42.724 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.721+0000 7f1f3d15a700 1 -- 172.21.15.159:0/3315724478 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1f381094a0 con 0x7f1f380ff550 2023-03-27T10:22:42.724 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.721+0000 7f1f2ffff700 1 -- 172.21.15.159:0/3315724478 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1f28007c60 con 0x7f1f380ff550 2023-03-27T10:22:42.724 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.721+0000 7f1f2ffff700 1 -- 172.21.15.159:0/3315724478 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1f28018660 con 0x7f1f380ff550 2023-03-27T10:22:42.725 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.721+0000 7f1f3d15a700 1 -- 172.21.15.159:0/3315724478 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1f38109310 con 0x7f1f380ff550 2023-03-27T10:22:42.725 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.722+0000 7f1f3d15a700 1 -- 172.21.15.159:0/3315724478 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1f3819dfe0 con 0x7f1f380ff550 2023-03-27T10:22:42.725 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.723+0000 7f1f2ffff700 1 -- 172.21.15.159:0/3315724478 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f1f28018880 con 0x7f1f380ff550 2023-03-27T10:22:42.726 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.723+0000 7f1f2ffff700 1 --2- 172.21.15.159:0/3315724478 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1f2409b3a0 0x7f1f2409d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:42.726 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.724+0000 7f1f36d9d700 1 --2- 172.21.15.159:0/3315724478 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1f2409b3a0 0x7f1f2409d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:42.726 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.724+0000 7f1f2ffff700 1 -- 172.21.15.159:0/3315724478 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(89..89 src has 1..89) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f1f280c5320 con 0x7f1f380ff550 2023-03-27T10:22:42.726 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.724+0000 7f1f36d9d700 1 --2- 172.21.15.159:0/3315724478 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1f2409b3a0 0x7f1f2409d850 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f1f2000b2e0 tx=0x7f1f20005fd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:42.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:42 smithi171 ceph-mon[94808]: pgmap v266: 129 pgs: 129 active+clean; 579 KiB data, 78 MiB used, 1.0 TiB / 1.0 TiB avail; 1.5 KiB/s wr, 4 op/s 2023-03-27T10:22:42.727 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.728+0000 7f1f2ffff700 1 -- 172.21.15.159:0/3315724478 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f1f28090a50 con 0x7f1f380ff550 2023-03-27T10:22:42.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:42 smithi161 ceph-mon[94333]: pgmap v266: 129 pgs: 129 active+clean; 579 KiB data, 78 MiB used, 1.0 TiB / 1.0 TiB avail; 1.5 KiB/s wr, 4 op/s 2023-03-27T10:22:42.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:42 smithi159 ceph-mon[93039]: pgmap v266: 129 pgs: 129 active+clean; 579 KiB data, 78 MiB used, 1.0 TiB / 1.0 TiB avail; 1.5 KiB/s wr, 4 op/s 2023-03-27T10:22:42.910 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.911+0000 7f1f3d15a700 1 -- 172.21.15.159:0/3315724478 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f1f3819e590 con 0x7f1f380ff550 2023-03-27T10:22:42.910 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.911+0000 7f1f2ffff700 1 -- 172.21.15.159:0/3315724478 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v89) v1 ==== 93+0+10 (secure 0 0 0) 0x7f1f28094070 con 0x7f1f380ff550 2023-03-27T10:22:42.911 INFO:teuthology.orchestra.run.smithi159.stdout:pg_num: 1 2023-03-27T10:22:42.913 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.913+0000 7f1f3d15a700 1 -- 172.21.15.159:0/3315724478 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1f2409b3a0 msgr2=0x7f1f2409d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:42.913 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.913+0000 7f1f3d15a700 1 --2- 172.21.15.159:0/3315724478 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1f2409b3a0 0x7f1f2409d850 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f1f2000b2e0 tx=0x7f1f20005fd0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.913 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.913+0000 7f1f3d15a700 1 -- 172.21.15.159:0/3315724478 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1f380ff550 msgr2=0x7f1f38195f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:42.913 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.913+0000 7f1f3d15a700 1 --2- 172.21.15.159:0/3315724478 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1f380ff550 0x7f1f38195f30 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f1f2800c9b0 tx=0x7f1f2800cd70 comp rx=0 tx=0).stop 2023-03-27T10:22:42.913 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.914+0000 7f1f3d15a700 1 -- 172.21.15.159:0/3315724478 shutdown_connections 2023-03-27T10:22:42.914 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.914+0000 7f1f3d15a700 1 --2- 172.21.15.159:0/3315724478 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f1f2409b3a0 0x7f1f2409d850 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.914 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.914+0000 7f1f3d15a700 1 --2- 172.21.15.159:0/3315724478 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f1f380fe360 0x7f1f381959f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.914 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.914+0000 7f1f3d15a700 1 --2- 172.21.15.159:0/3315724478 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f1f380ffee0 0x7f1f3819cfa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.914 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.914+0000 7f1f3d15a700 1 --2- 172.21.15.159:0/3315724478 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f1f380ff550 0x7f1f38195f30 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:42.914 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.914+0000 7f1f3d15a700 1 -- 172.21.15.159:0/3315724478 >> 172.21.15.159:0/3315724478 conn(0x7f1f380f99a0 msgr2=0x7f1f380fbb30 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:42.914 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.914+0000 7f1f3d15a700 1 -- 172.21.15.159:0/3315724478 shutdown_connections 2023-03-27T10:22:42.914 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:42.914+0000 7f1f3d15a700 1 -- 172.21.15.159:0/3315724478 wait complete. 2023-03-27T10:22:42.925 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2023-03-27T10:22:43.050 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.050+0000 7f0260210700 1 -- 172.21.15.159:0/2457619335 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f02580ff790 msgr2=0x7f0258107040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:43.050 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.050+0000 7f0260210700 1 --2- 172.21.15.159:0/2457619335 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f02580ff790 0x7f0258107040 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f025000b0a0 tx=0x7f025000b3b0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.050 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.051+0000 7f0260210700 1 -- 172.21.15.159:0/2457619335 shutdown_connections 2023-03-27T10:22:43.050 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.051+0000 7f0260210700 1 --2- 172.21.15.159:0/2457619335 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f02580ff790 0x7f0258107040 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.051 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.051+0000 7f0260210700 1 --2- 172.21.15.159:0/2457619335 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f0258100190 0x7f02581005a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.051 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.051+0000 7f0260210700 1 --2- 172.21.15.159:0/2457619335 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f02580fede0 0x7f02580ff250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.051 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.051+0000 7f0260210700 1 -- 172.21.15.159:0/2457619335 >> 172.21.15.159:0/2457619335 conn(0x7f02580f99a0 msgr2=0x7f02580fbdb0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:43.051 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.051+0000 7f0260210700 1 -- 172.21.15.159:0/2457619335 shutdown_connections 2023-03-27T10:22:43.051 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.051+0000 7f0260210700 1 -- 172.21.15.159:0/2457619335 wait complete. 2023-03-27T10:22:43.053 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.051+0000 7f0260210700 1 Processor -- start 2023-03-27T10:22:43.053 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.051+0000 7f0260210700 1 -- start start 2023-03-27T10:22:43.053 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.051+0000 7f0260210700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f02580fede0 0x7f02580fe840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:43.054 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.051+0000 7f0260210700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f02580ff790 0x7f025819a830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:43.054 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.051+0000 7f0260210700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0258100190 0x7f025819ce70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:43.054 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.051+0000 7f0260210700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f025819d610 con 0x7f0258100190 2023-03-27T10:22:43.054 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.051+0000 7f0260210700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f025819d780 con 0x7f02580ff790 2023-03-27T10:22:43.054 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.051+0000 7f0260210700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f025819d940 con 0x7f02580fede0 2023-03-27T10:22:43.054 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f025e7ad700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0258100190 0x7f025819ce70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:43.054 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f025e7ad700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0258100190 0x7f025819ce70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45854/0 (socket says 172.21.15.159:45854) 2023-03-27T10:22:43.055 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f025e7ad700 1 -- 172.21.15.159:0/2981038957 learned_addr learned my addr 172.21.15.159:0/2981038957 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:43.055 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f025d7ab700 1 --2- 172.21.15.159:0/2981038957 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f02580ff790 0x7f025819a830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:43.055 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f025dfac700 1 --2- 172.21.15.159:0/2981038957 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f02580fede0 0x7f02580fe840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:43.055 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f025e7ad700 1 -- 172.21.15.159:0/2981038957 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f02580fede0 msgr2=0x7f02580fe840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:43.055 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f025e7ad700 1 --2- 172.21.15.159:0/2981038957 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f02580fede0 0x7f02580fe840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.055 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f025e7ad700 1 -- 172.21.15.159:0/2981038957 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f02580ff790 msgr2=0x7f025819a830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:43.055 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f025e7ad700 1 --2- 172.21.15.159:0/2981038957 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f02580ff790 0x7f025819a830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.055 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f025e7ad700 1 -- 172.21.15.159:0/2981038957 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0250009d00 con 0x7f0258100190 2023-03-27T10:22:43.056 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f025d7ab700 1 --2- 172.21.15.159:0/2981038957 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f02580ff790 0x7f025819a830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-03-27T10:22:43.056 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f025e7ad700 1 --2- 172.21.15.159:0/2981038957 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0258100190 0x7f025819ce70 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f025000b780 tx=0x7f025000be10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:43.056 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f024effd700 1 -- 172.21.15.159:0/2981038957 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f025000e040 con 0x7f0258100190 2023-03-27T10:22:43.056 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f024effd700 1 -- 172.21.15.159:0/2981038957 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0250004e60 con 0x7f0258100190 2023-03-27T10:22:43.056 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f0260210700 1 -- 172.21.15.159:0/2981038957 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f025810a520 con 0x7f0258100190 2023-03-27T10:22:43.056 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f024effd700 1 -- 172.21.15.159:0/2981038957 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02500053b0 con 0x7f0258100190 2023-03-27T10:22:43.056 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.052+0000 7f0260210700 1 -- 172.21.15.159:0/2981038957 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f025810a390 con 0x7f0258100190 2023-03-27T10:22:43.056 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.053+0000 7f0260210700 1 -- 172.21.15.159:0/2981038957 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0258105b30 con 0x7f0258100190 2023-03-27T10:22:43.057 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.054+0000 7f024effd700 1 -- 172.21.15.159:0/2981038957 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f0250005510 con 0x7f0258100190 2023-03-27T10:22:43.057 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.055+0000 7f024effd700 1 --2- 172.21.15.159:0/2981038957 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f024409b420 0x7f024409d8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:43.057 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.055+0000 7f025dfac700 1 --2- 172.21.15.159:0/2981038957 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f024409b420 0x7f024409d8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:43.057 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.055+0000 7f024effd700 1 -- 172.21.15.159:0/2981038957 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(89..89 src has 1..89) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f02500c6a30 con 0x7f0258100190 2023-03-27T10:22:43.058 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.055+0000 7f025dfac700 1 --2- 172.21.15.159:0/2981038957 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f024409b420 0x7f024409d8d0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f0254009b30 tx=0x7f0254009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:43.058 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.059+0000 7f024effd700 1 -- 172.21.15.159:0/2981038957 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f0250092190 con 0x7f0258100190 2023-03-27T10:22:43.242 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.243+0000 7f0260210700 1 -- 172.21.15.159:0/2981038957 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f0258066c50 con 0x7f0258100190 2023-03-27T10:22:43.242 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.243+0000 7f024effd700 1 -- 172.21.15.159:0/2981038957 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v89) v1 ==== 104+0+11 (secure 0 0 0) 0x7f02500957b0 con 0x7f0258100190 2023-03-27T10:22:43.243 INFO:teuthology.orchestra.run.smithi159.stdout:pg_num: 32 2023-03-27T10:22:43.245 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.245+0000 7f0260210700 1 -- 172.21.15.159:0/2981038957 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f024409b420 msgr2=0x7f024409d8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:43.245 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.245+0000 7f0260210700 1 --2- 172.21.15.159:0/2981038957 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f024409b420 0x7f024409d8d0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f0254009b30 tx=0x7f0254009260 comp rx=0 tx=0).stop 2023-03-27T10:22:43.245 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.245+0000 7f0260210700 1 -- 172.21.15.159:0/2981038957 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0258100190 msgr2=0x7f025819ce70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:43.245 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.245+0000 7f0260210700 1 --2- 172.21.15.159:0/2981038957 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0258100190 0x7f025819ce70 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f025000b780 tx=0x7f025000be10 comp rx=0 tx=0).stop 2023-03-27T10:22:43.245 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.245+0000 7f0260210700 1 -- 172.21.15.159:0/2981038957 shutdown_connections 2023-03-27T10:22:43.245 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.245+0000 7f0260210700 1 --2- 172.21.15.159:0/2981038957 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f024409b420 0x7f024409d8d0 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.246 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.245+0000 7f0260210700 1 --2- 172.21.15.159:0/2981038957 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f02580fede0 0x7f02580fe840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.246 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.245+0000 7f0260210700 1 --2- 172.21.15.159:0/2981038957 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f0258100190 0x7f025819ce70 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.246 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.245+0000 7f0260210700 1 --2- 172.21.15.159:0/2981038957 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f02580ff790 0x7f025819a830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.246 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.245+0000 7f0260210700 1 -- 172.21.15.159:0/2981038957 >> 172.21.15.159:0/2981038957 conn(0x7f02580f99a0 msgr2=0x7f02580fbc10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:43.246 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.245+0000 7f0260210700 1 -- 172.21.15.159:0/2981038957 shutdown_connections 2023-03-27T10:22:43.246 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.245+0000 7f0260210700 1 -- 172.21.15.159:0/2981038957 wait complete. 2023-03-27T10:22:43.256 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2023-03-27T10:22:43.382 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.380+0000 7f70eaea6700 1 -- 172.21.15.159:0/2861403939 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f70e40ffe60 msgr2=0x7f70e41002d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:43.382 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.380+0000 7f70eaea6700 1 --2- 172.21.15.159:0/2861403939 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f70e40ffe60 0x7f70e41002d0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f70d0009990 tx=0x7f70d0009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.382 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70eaea6700 1 -- 172.21.15.159:0/2861403939 shutdown_connections 2023-03-27T10:22:43.382 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70eaea6700 1 --2- 172.21.15.159:0/2861403939 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f70e40febd0 0x7f70e40fefe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.382 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70eaea6700 1 --2- 172.21.15.159:0/2861403939 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f70e4100810 0x7f70e41070b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.383 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70eaea6700 1 --2- 172.21.15.159:0/2861403939 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f70e40ffe60 0x7f70e41002d0 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.383 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70eaea6700 1 -- 172.21.15.159:0/2861403939 >> 172.21.15.159:0/2861403939 conn(0x7f70e40fa210 msgr2=0x7f70e40fc620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:43.383 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70eaea6700 1 -- 172.21.15.159:0/2861403939 shutdown_connections 2023-03-27T10:22:43.383 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70eaea6700 1 -- 172.21.15.159:0/2861403939 wait complete. 2023-03-27T10:22:43.383 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70eaea6700 1 Processor -- start 2023-03-27T10:22:43.383 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70eaea6700 1 -- start start 2023-03-27T10:22:43.383 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70eaea6700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f70e40febd0 0x7f70e4195a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:43.383 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70eaea6700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f70e40ffe60 0x7f70e4195f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:43.384 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70eaea6700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f70e4100810 0x7f70e419cff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:43.384 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70eaea6700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70e419d5c0 con 0x7f70e4100810 2023-03-27T10:22:43.384 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70eaea6700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70e419d700 con 0x7f70e40febd0 2023-03-27T10:22:43.384 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70eaea6700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70e419d890 con 0x7f70e40ffe60 2023-03-27T10:22:43.384 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70e9443700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f70e4100810 0x7f70e419cff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:43.384 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.381+0000 7f70e9443700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f70e4100810 0x7f70e419cff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45868/0 (socket says 172.21.15.159:45868) 2023-03-27T10:22:43.384 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.382+0000 7f70e9443700 1 -- 172.21.15.159:0/2949905518 learned_addr learned my addr 172.21.15.159:0/2949905518 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:43.385 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.382+0000 7f70e3fff700 1 --2- 172.21.15.159:0/2949905518 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f70e40ffe60 0x7f70e4195f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:43.385 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.382+0000 7f70e8c42700 1 --2- 172.21.15.159:0/2949905518 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f70e40febd0 0x7f70e4195a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:43.385 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.382+0000 7f70e9443700 1 -- 172.21.15.159:0/2949905518 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f70e40ffe60 msgr2=0x7f70e4195f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:43.385 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.382+0000 7f70e9443700 1 --2- 172.21.15.159:0/2949905518 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f70e40ffe60 0x7f70e4195f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.385 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.382+0000 7f70e9443700 1 -- 172.21.15.159:0/2949905518 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f70e40febd0 msgr2=0x7f70e4195a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:43.385 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.382+0000 7f70e9443700 1 --2- 172.21.15.159:0/2949905518 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f70e40febd0 0x7f70e4195a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.385 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.382+0000 7f70e9443700 1 -- 172.21.15.159:0/2949905518 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f70d00095f0 con 0x7f70e4100810 2023-03-27T10:22:43.385 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.382+0000 7f70e3fff700 1 --2- 172.21.15.159:0/2949905518 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f70e40ffe60 0x7f70e4195f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:22:43.386 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.382+0000 7f70e9443700 1 --2- 172.21.15.159:0/2949905518 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f70e4100810 0x7f70e419cff0 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f70dc00ef30 tx=0x7f70dc00c450 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:43.386 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.382+0000 7f70e1ffb700 1 -- 172.21.15.159:0/2949905518 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f70dc018070 con 0x7f70e4100810 2023-03-27T10:22:43.386 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.382+0000 7f70e1ffb700 1 -- 172.21.15.159:0/2949905518 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f70dc00f040 con 0x7f70e4100810 2023-03-27T10:22:43.386 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.382+0000 7f70eaea6700 1 -- 172.21.15.159:0/2949905518 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f70e410a530 con 0x7f70e4100810 2023-03-27T10:22:43.386 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.382+0000 7f70e1ffb700 1 -- 172.21.15.159:0/2949905518 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f70dc003dc0 con 0x7f70e4100810 2023-03-27T10:22:43.386 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.382+0000 7f70eaea6700 1 -- 172.21.15.159:0/2949905518 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f70e410a3a0 con 0x7f70e4100810 2023-03-27T10:22:43.387 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.383+0000 7f70eaea6700 1 -- 172.21.15.159:0/2949905518 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f70e40674c0 con 0x7f70e4100810 2023-03-27T10:22:43.387 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.384+0000 7f70e1ffb700 1 -- 172.21.15.159:0/2949905518 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f70dc00caa0 con 0x7f70e4100810 2023-03-27T10:22:43.387 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.385+0000 7f70e1ffb700 1 --2- 172.21.15.159:0/2949905518 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f70d409b3a0 0x7f70d409d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:43.387 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.385+0000 7f70e8c42700 1 --2- 172.21.15.159:0/2949905518 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f70d409b3a0 0x7f70d409d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:43.388 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.385+0000 7f70e1ffb700 1 -- 172.21.15.159:0/2949905518 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(89..89 src has 1..89) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f70dc0c4fc0 con 0x7f70e4100810 2023-03-27T10:22:43.388 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.385+0000 7f70e8c42700 1 --2- 172.21.15.159:0/2949905518 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f70d409b3a0 0x7f70d409d850 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f70d8009bf0 tx=0x7f70d8009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:43.388 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.389+0000 7f70e1ffb700 1 -- 172.21.15.159:0/2949905518 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f70dc090720 con 0x7f70e4100810 2023-03-27T10:22:43.574 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.575+0000 7f70eaea6700 1 -- 172.21.15.159:0/2949905518 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f70e419e630 con 0x7f70e4100810 2023-03-27T10:22:43.575 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.575+0000 7f70e1ffb700 1 -- 172.21.15.159:0/2949905518 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v89) v1 ==== 100+0+11 (secure 0 0 0) 0x7f70dc093d40 con 0x7f70e4100810 2023-03-27T10:22:43.575 INFO:teuthology.orchestra.run.smithi159.stdout:pg_num: 64 2023-03-27T10:22:43.577 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.577+0000 7f70eaea6700 1 -- 172.21.15.159:0/2949905518 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f70d409b3a0 msgr2=0x7f70d409d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:43.577 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.577+0000 7f70eaea6700 1 --2- 172.21.15.159:0/2949905518 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f70d409b3a0 0x7f70d409d850 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f70d8009bf0 tx=0x7f70d8009260 comp rx=0 tx=0).stop 2023-03-27T10:22:43.577 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.577+0000 7f70eaea6700 1 -- 172.21.15.159:0/2949905518 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f70e4100810 msgr2=0x7f70e419cff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:43.577 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.577+0000 7f70eaea6700 1 --2- 172.21.15.159:0/2949905518 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f70e4100810 0x7f70e419cff0 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f70dc00ef30 tx=0x7f70dc00c450 comp rx=0 tx=0).stop 2023-03-27T10:22:43.578 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.577+0000 7f70eaea6700 1 -- 172.21.15.159:0/2949905518 shutdown_connections 2023-03-27T10:22:43.578 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.577+0000 7f70eaea6700 1 --2- 172.21.15.159:0/2949905518 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f70d409b3a0 0x7f70d409d850 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.578 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.577+0000 7f70eaea6700 1 --2- 172.21.15.159:0/2949905518 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f70e4100810 0x7f70e419cff0 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.578 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.577+0000 7f70eaea6700 1 --2- 172.21.15.159:0/2949905518 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f70e40febd0 0x7f70e4195a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.578 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.577+0000 7f70eaea6700 1 --2- 172.21.15.159:0/2949905518 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f70e40ffe60 0x7f70e4195f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.578 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.577+0000 7f70eaea6700 1 -- 172.21.15.159:0/2949905518 >> 172.21.15.159:0/2949905518 conn(0x7f70e40fa210 msgr2=0x7f70e40678e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:43.578 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.577+0000 7f70eaea6700 1 -- 172.21.15.159:0/2949905518 shutdown_connections 2023-03-27T10:22:43.578 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.578+0000 7f70eaea6700 1 -- 172.21.15.159:0/2949905518 wait complete. 2023-03-27T10:22:43.588 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:43 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/1298482205' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:22:43.588 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:43 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3315724478' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-03-27T10:22:43.588 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:43 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/2981038957' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-03-27T10:22:43.589 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2023-03-27T10:22:43.713 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.713+0000 7f4dd9851700 1 -- 172.21.15.159:0/3849200005 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4dd40feea0 msgr2=0x7f4dd40ff310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:43.713 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.713+0000 7f4dd9851700 1 --2- 172.21.15.159:0/3849200005 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4dd40feea0 0x7f4dd40ff310 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f4dc8009990 tx=0x7f4dc8009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.713 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.714+0000 7f4dd9851700 1 -- 172.21.15.159:0/3849200005 shutdown_connections 2023-03-27T10:22:43.713 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.714+0000 7f4dd9851700 1 --2- 172.21.15.159:0/3849200005 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f4dd40ff850 0x7f4dd4107100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.714 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.714+0000 7f4dd9851700 1 --2- 172.21.15.159:0/3849200005 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f4dd4100250 0x7f4dd4100660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.714 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.714+0000 7f4dd9851700 1 --2- 172.21.15.159:0/3849200005 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4dd40feea0 0x7f4dd40ff310 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.714 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.714+0000 7f4dd9851700 1 -- 172.21.15.159:0/3849200005 >> 172.21.15.159:0/3849200005 conn(0x7f4dd40f9a40 msgr2=0x7f4dd40fbe70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:43.714 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.714+0000 7f4dd9851700 1 -- 172.21.15.159:0/3849200005 shutdown_connections 2023-03-27T10:22:43.714 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.714+0000 7f4dd9851700 1 -- 172.21.15.159:0/3849200005 wait complete. 2023-03-27T10:22:43.716 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.714+0000 7f4dd9851700 1 Processor -- start 2023-03-27T10:22:43.716 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.714+0000 7f4dd9851700 1 -- start start 2023-03-27T10:22:43.716 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.714+0000 7f4dd9851700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4dd40feea0 0x7f4dd40fe920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:43.716 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.714+0000 7f4dd9851700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f4dd40ff850 0x7f4dd419a770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:43.716 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.714+0000 7f4dd9851700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f4dd4100250 0x7f4dd419cdc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:43.717 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.714+0000 7f4dd9851700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4dd419d590 con 0x7f4dd40feea0 2023-03-27T10:22:43.717 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.714+0000 7f4dd9851700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4dd419d700 con 0x7f4dd4100250 2023-03-27T10:22:43.717 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.714+0000 7f4dd9851700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4dd419d8c0 con 0x7f4dd40ff850 2023-03-27T10:22:43.717 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.715+0000 7f4dd2ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4dd40feea0 0x7f4dd40fe920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:43.717 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.715+0000 7f4dd2ffd700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4dd40feea0 0x7f4dd40fe920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45878/0 (socket says 172.21.15.159:45878) 2023-03-27T10:22:43.717 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.715+0000 7f4dd2ffd700 1 -- 172.21.15.159:0/3660026272 learned_addr learned my addr 172.21.15.159:0/3660026272 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:43.717 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.715+0000 7f4dd27fc700 1 --2- 172.21.15.159:0/3660026272 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f4dd40ff850 0x7f4dd419a770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:43.717 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.715+0000 7f4dd2ffd700 1 -- 172.21.15.159:0/3660026272 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f4dd40ff850 msgr2=0x7f4dd419a770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:43.718 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.715+0000 7f4dd2ffd700 1 --2- 172.21.15.159:0/3660026272 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f4dd40ff850 0x7f4dd419a770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.718 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.715+0000 7f4dd2ffd700 1 -- 172.21.15.159:0/3660026272 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f4dd4100250 msgr2=0x7f4dd419cdc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:43.718 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.715+0000 7f4dd2ffd700 1 --2- 172.21.15.159:0/3660026272 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f4dd4100250 0x7f4dd419cdc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.718 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.715+0000 7f4dd2ffd700 1 -- 172.21.15.159:0/3660026272 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4dc80095f0 con 0x7f4dd40feea0 2023-03-27T10:22:43.718 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.715+0000 7f4dd2ffd700 1 --2- 172.21.15.159:0/3660026272 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4dd40feea0 0x7f4dd40fe920 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f4dc400c920 tx=0x7f4dc400cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:43.718 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.715+0000 7f4dd884f700 1 -- 172.21.15.159:0/3660026272 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dc4007b00 con 0x7f4dd40feea0 2023-03-27T10:22:43.718 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.715+0000 7f4dd9851700 1 -- 172.21.15.159:0/3660026272 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4dd410a5e0 con 0x7f4dd40feea0 2023-03-27T10:22:43.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.715+0000 7f4dd884f700 1 -- 172.21.15.159:0/3660026272 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4dc4004cf0 con 0x7f4dd40feea0 2023-03-27T10:22:43.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.715+0000 7f4dd884f700 1 -- 172.21.15.159:0/3660026272 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dc4005520 con 0x7f4dd40feea0 2023-03-27T10:22:43.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.715+0000 7f4dd9851700 1 -- 172.21.15.159:0/3660026272 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4dd410a450 con 0x7f4dd40feea0 2023-03-27T10:22:43.719 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.716+0000 7f4dd9851700 1 -- 172.21.15.159:0/3660026272 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4dd419dfb0 con 0x7f4dd40feea0 2023-03-27T10:22:43.720 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.718+0000 7f4dd884f700 1 -- 172.21.15.159:0/3660026272 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f4dc401f070 con 0x7f4dd40feea0 2023-03-27T10:22:43.720 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.718+0000 7f4dd884f700 1 --2- 172.21.15.159:0/3660026272 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f4dbc09b3a0 0x7f4dbc09d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:43.720 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.718+0000 7f4dd884f700 1 -- 172.21.15.159:0/3660026272 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(89..89 src has 1..89) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f4dc40c5010 con 0x7f4dd40feea0 2023-03-27T10:22:43.720 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.718+0000 7f4dd27fc700 1 --2- 172.21.15.159:0/3660026272 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f4dbc09b3a0 0x7f4dbc09d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:43.720 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.719+0000 7f4dd27fc700 1 --2- 172.21.15.159:0/3660026272 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f4dbc09b3a0 0x7f4dbc09d850 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f4dc8006010 tx=0x7f4dc800b540 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:43.721 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.722+0000 7f4dd884f700 1 -- 172.21.15.159:0/3660026272 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f4dc4090740 con 0x7f4dd40feea0 2023-03-27T10:22:43.725 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:43 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/1298482205' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:22:43.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:43 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3315724478' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-03-27T10:22:43.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:43 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/2981038957' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-03-27T10:22:43.833 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:43 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/1298482205' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-03-27T10:22:43.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:43 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3315724478' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-03-27T10:22:43.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:43 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/2981038957' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-03-27T10:22:43.908 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.908+0000 7f4dd9851700 1 -- 172.21.15.159:0/3660026272 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) v1 -- 0x7f4dd4105c40 con 0x7f4dd40feea0 2023-03-27T10:22:43.908 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.909+0000 7f4dd884f700 1 -- 172.21.15.159:0/3660026272 <== mon.0 v2:172.21.15.159:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v89) v1 ==== 103+0+11 (secure 0 0 0) 0x7f4dc401d070 con 0x7f4dd40feea0 2023-03-27T10:22:43.908 INFO:teuthology.orchestra.run.smithi159.stdout:pg_num: 32 2023-03-27T10:22:43.910 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.911+0000 7f4dd9851700 1 -- 172.21.15.159:0/3660026272 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f4dbc09b3a0 msgr2=0x7f4dbc09d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:43.910 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.911+0000 7f4dd9851700 1 --2- 172.21.15.159:0/3660026272 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f4dbc09b3a0 0x7f4dbc09d850 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f4dc8006010 tx=0x7f4dc800b540 comp rx=0 tx=0).stop 2023-03-27T10:22:43.911 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.911+0000 7f4dd9851700 1 -- 172.21.15.159:0/3660026272 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4dd40feea0 msgr2=0x7f4dd40fe920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:43.911 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.911+0000 7f4dd9851700 1 --2- 172.21.15.159:0/3660026272 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4dd40feea0 0x7f4dd40fe920 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f4dc400c920 tx=0x7f4dc400cce0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.911 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.911+0000 7f4dd9851700 1 -- 172.21.15.159:0/3660026272 shutdown_connections 2023-03-27T10:22:43.911 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.911+0000 7f4dd9851700 1 --2- 172.21.15.159:0/3660026272 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f4dbc09b3a0 0x7f4dbc09d850 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.911 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.911+0000 7f4dd9851700 1 --2- 172.21.15.159:0/3660026272 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f4dd4100250 0x7f4dd419cdc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.911 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.911+0000 7f4dd9851700 1 --2- 172.21.15.159:0/3660026272 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f4dd40feea0 0x7f4dd40fe920 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.911 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.911+0000 7f4dd9851700 1 --2- 172.21.15.159:0/3660026272 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f4dd40ff850 0x7f4dd419a770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:43.911 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.911+0000 7f4dd9851700 1 -- 172.21.15.159:0/3660026272 >> 172.21.15.159:0/3660026272 conn(0x7f4dd40f9a40 msgr2=0x7f4dd40fbcd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:43.912 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.911+0000 7f4dd9851700 1 -- 172.21.15.159:0/3660026272 shutdown_connections 2023-03-27T10:22:43.912 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:43.911+0000 7f4dd9851700 1 -- 172.21.15.159:0/3660026272 wait complete. 2023-03-27T10:22:43.922 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph 'osd blocklist ls' 2023-03-27T10:22:44.042 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.043+0000 7f9774541700 1 -- 172.21.15.159:0/3138770380 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f976c100810 msgr2=0x7f976c1070b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:44.042 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.043+0000 7f9774541700 1 --2- 172.21.15.159:0/3138770380 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f976c100810 0x7f976c1070b0 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f976400b0a0 tx=0x7f976400b3b0 comp rx=0 tx=0).stop 2023-03-27T10:22:44.042 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.043+0000 7f9774541700 1 -- 172.21.15.159:0/3138770380 shutdown_connections 2023-03-27T10:22:44.043 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.043+0000 7f9774541700 1 --2- 172.21.15.159:0/3138770380 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f976c100810 0x7f976c1070b0 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:44.043 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.043+0000 7f9774541700 1 --2- 172.21.15.159:0/3138770380 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f976c0febd0 0x7f976c0fefe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:44.043 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.043+0000 7f9774541700 1 --2- 172.21.15.159:0/3138770380 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f976c0ffe60 0x7f976c1002d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:44.043 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.043+0000 7f9774541700 1 -- 172.21.15.159:0/3138770380 >> 172.21.15.159:0/3138770380 conn(0x7f976c0fa210 msgr2=0x7f976c0fc620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:44.043 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.043+0000 7f9774541700 1 -- 172.21.15.159:0/3138770380 shutdown_connections 2023-03-27T10:22:44.043 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.043+0000 7f9774541700 1 -- 172.21.15.159:0/3138770380 wait complete. 2023-03-27T10:22:44.045 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.043+0000 7f9774541700 1 Processor -- start 2023-03-27T10:22:44.045 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.043+0000 7f9774541700 1 -- start start 2023-03-27T10:22:44.045 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9774541700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f976c0febd0 0x7f976c1949b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:44.046 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9774541700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f976c0ffe60 0x7f976c194ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:44.046 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9774541700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f976c100810 0x7f976c19bf60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:44.046 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9774541700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f976c19c530 con 0x7f976c100810 2023-03-27T10:22:44.046 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9774541700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f976c19c670 con 0x7f976c0ffe60 2023-03-27T10:22:44.046 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9774541700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f976c19c800 con 0x7f976c0febd0 2023-03-27T10:22:44.046 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9772ade700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f976c100810 0x7f976c19bf60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:44.046 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9772ade700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f976c100810 0x7f976c19bf60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45896/0 (socket says 172.21.15.159:45896) 2023-03-27T10:22:44.046 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9772ade700 1 -- 172.21.15.159:0/4170514276 learned_addr learned my addr 172.21.15.159:0/4170514276 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:44.047 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f97722dd700 1 --2- 172.21.15.159:0/4170514276 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f976c0febd0 0x7f976c1949b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:44.047 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9772ade700 1 -- 172.21.15.159:0/4170514276 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f976c0febd0 msgr2=0x7f976c1949b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:44.047 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9772ade700 1 --2- 172.21.15.159:0/4170514276 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f976c0febd0 0x7f976c1949b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:44.047 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9772ade700 1 -- 172.21.15.159:0/4170514276 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f976c0ffe60 msgr2=0x7f976c194ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:44.047 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9771adc700 1 --2- 172.21.15.159:0/4170514276 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f976c0ffe60 0x7f976c194ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:44.047 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9772ade700 1 --2- 172.21.15.159:0/4170514276 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f976c0ffe60 0x7f976c194ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:44.048 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9772ade700 1 -- 172.21.15.159:0/4170514276 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9764009d00 con 0x7f976c100810 2023-03-27T10:22:44.048 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f97722dd700 1 --2- 172.21.15.159:0/4170514276 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f976c0febd0 0x7f976c1949b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-03-27T10:22:44.048 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9771adc700 1 --2- 172.21.15.159:0/4170514276 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f976c0ffe60 0x7f976c194ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-03-27T10:22:44.048 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f9772ade700 1 --2- 172.21.15.159:0/4170514276 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f976c100810 0x7f976c19bf60 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f976400b070 tx=0x7f9764004db0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:44.048 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.044+0000 7f97637fe700 1 -- 172.21.15.159:0/4170514276 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f976400e040 con 0x7f976c100810 2023-03-27T10:22:44.048 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.045+0000 7f97637fe700 1 -- 172.21.15.159:0/4170514276 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f976401bd20 con 0x7f976c100810 2023-03-27T10:22:44.048 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.045+0000 7f9774541700 1 -- 172.21.15.159:0/4170514276 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f976c10a530 con 0x7f976c100810 2023-03-27T10:22:44.049 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.045+0000 7f97637fe700 1 -- 172.21.15.159:0/4170514276 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9764005910 con 0x7f976c100810 2023-03-27T10:22:44.049 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.045+0000 7f9774541700 1 -- 172.21.15.159:0/4170514276 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f976c10a3a0 con 0x7f976c100810 2023-03-27T10:22:44.049 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.045+0000 7f9774541700 1 -- 172.21.15.159:0/4170514276 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f976c105b50 con 0x7f976c100810 2023-03-27T10:22:44.049 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.047+0000 7f97637fe700 1 -- 172.21.15.159:0/4170514276 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f976400bd70 con 0x7f976c100810 2023-03-27T10:22:44.050 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.047+0000 7f97637fe700 1 --2- 172.21.15.159:0/4170514276 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f975809b3a0 0x7f975809d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:44.050 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.047+0000 7f97722dd700 1 --2- 172.21.15.159:0/4170514276 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f975809b3a0 0x7f975809d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:44.050 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.047+0000 7f97637fe700 1 -- 172.21.15.159:0/4170514276 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(89..89 src has 1..89) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f97640c7570 con 0x7f976c100810 2023-03-27T10:22:44.050 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.047+0000 7f97722dd700 1 --2- 172.21.15.159:0/4170514276 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f975809b3a0 0x7f975809d850 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f9768005fd0 tx=0x7f9768005e20 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:44.050 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.050+0000 7f97637fe700 1 -- 172.21.15.159:0/4170514276 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f9764092d30 con 0x7f976c100810 2023-03-27T10:22:44.462 INFO:teuthology.orchestra.run.smithi159.stderr:no valid command found; 10 closest matches: 2023-03-27T10:22:44.463 INFO:teuthology.orchestra.run.smithi159.stderr:pg stat 2023-03-27T10:22:44.463 INFO:teuthology.orchestra.run.smithi159.stderr:pg getmap 2023-03-27T10:22:44.463 INFO:teuthology.orchestra.run.smithi159.stderr:pg dump [...] 2023-03-27T10:22:44.463 INFO:teuthology.orchestra.run.smithi159.stderr:pg dump_json [...] 2023-03-27T10:22:44.463 INFO:teuthology.orchestra.run.smithi159.stderr:pg dump_pools_json 2023-03-27T10:22:44.464 INFO:teuthology.orchestra.run.smithi159.stderr:pg ls-by-pool [...] 2023-03-27T10:22:44.464 INFO:teuthology.orchestra.run.smithi159.stderr:pg ls-by-primary [] [...] 2023-03-27T10:22:44.464 INFO:teuthology.orchestra.run.smithi159.stderr:pg ls-by-osd [] [...] 2023-03-27T10:22:44.464 INFO:teuthology.orchestra.run.smithi159.stderr:pg ls [] [...] 2023-03-27T10:22:44.464 INFO:teuthology.orchestra.run.smithi159.stderr:pg dump_stuck [...] [] 2023-03-27T10:22:44.464 INFO:teuthology.orchestra.run.smithi159.stderr:Error EINVAL: invalid command 2023-03-27T10:22:44.465 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-03-27T10:22:44.466 DEBUG:teuthology.orchestra.run.smithi159:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph 'osd blacklist ls' 2023-03-27T10:22:44.587 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.587+0000 7f55763b7700 1 -- 172.21.15.159:0/1699966835 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5570100800 msgr2=0x7f5570102be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:44.587 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.587+0000 7f55763b7700 1 --2- 172.21.15.159:0/1699966835 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5570100800 0x7f5570102be0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f5560009990 tx=0x7f5560009ca0 comp rx=0 tx=0).stop 2023-03-27T10:22:44.587 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.587+0000 7f55763b7700 1 -- 172.21.15.159:0/1699966835 shutdown_connections 2023-03-27T10:22:44.587 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.587+0000 7f55763b7700 1 --2- 172.21.15.159:0/1699966835 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f5570105a40 0x7f5570107e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:44.588 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.587+0000 7f55763b7700 1 --2- 172.21.15.159:0/1699966835 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5570100800 0x7f5570102be0 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:44.588 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.587+0000 7f55763b7700 1 --2- 172.21.15.159:0/1699966835 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f5570103120 0x7f5570105500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:44.588 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.587+0000 7f55763b7700 1 -- 172.21.15.159:0/1699966835 >> 172.21.15.159:0/1699966835 conn(0x7f55700fa1e0 msgr2=0x7f55700fc650 unknown :-1 s=STATE_NONE l=0).mark_down 2023-03-27T10:22:44.588 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.587+0000 7f55763b7700 1 -- 172.21.15.159:0/1699966835 shutdown_connections 2023-03-27T10:22:44.588 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.587+0000 7f55763b7700 1 -- 172.21.15.159:0/1699966835 wait complete. 2023-03-27T10:22:44.590 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.588+0000 7f55763b7700 1 Processor -- start 2023-03-27T10:22:44.590 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.588+0000 7f55763b7700 1 -- start start 2023-03-27T10:22:44.591 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f55763b7700 1 --2- >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f5570100800 0x7f5570195a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:44.591 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f55763b7700 1 --2- >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f5570103120 0x7f5570195f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:44.591 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f55763b7700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5570105a40 0x7f557019cff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:44.591 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f55763b7700 1 -- --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f557019d5c0 con 0x7f5570105a40 2023-03-27T10:22:44.591 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f55763b7700 1 -- --> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f557019d700 con 0x7f5570100800 2023-03-27T10:22:44.591 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f55763b7700 1 -- --> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f557019d890 con 0x7f5570103120 2023-03-27T10:22:44.591 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f5574954700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5570105a40 0x7f557019cff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:44.592 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f5574954700 1 --2- >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5570105a40 0x7f557019cff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.159:3300/0 says I am v2:172.21.15.159:45910/0 (socket says 172.21.15.159:45910) 2023-03-27T10:22:44.592 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f5574954700 1 -- 172.21.15.159:0/1223711869 learned_addr learned my addr 172.21.15.159:0/1223711869 (peer_addr_for_me v2:172.21.15.159:0/0) 2023-03-27T10:22:44.592 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f556ffff700 1 --2- 172.21.15.159:0/1223711869 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f5570100800 0x7f5570195a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:44.592 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f556f7fe700 1 --2- 172.21.15.159:0/1223711869 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f5570103120 0x7f5570195f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:44.592 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f5574954700 1 -- 172.21.15.159:0/1223711869 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f5570103120 msgr2=0x7f5570195f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:44.593 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f5574954700 1 --2- 172.21.15.159:0/1223711869 >> [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] conn(0x7f5570103120 0x7f5570195f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:44.593 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f5574954700 1 -- 172.21.15.159:0/1223711869 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f5570100800 msgr2=0x7f5570195a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-03-27T10:22:44.593 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f5574954700 1 --2- 172.21.15.159:0/1223711869 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f5570100800 0x7f5570195a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-03-27T10:22:44.594 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f5574954700 1 -- 172.21.15.159:0/1223711869 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f55600095f0 con 0x7f5570105a40 2023-03-27T10:22:44.594 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f556ffff700 1 --2- 172.21.15.159:0/1223711869 >> [v2:172.21.15.161:3300/0,v1:172.21.15.161:6789/0] conn(0x7f5570100800 0x7f5570195a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-03-27T10:22:44.594 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f5574954700 1 --2- 172.21.15.159:0/1223711869 >> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] conn(0x7f5570105a40 0x7f557019cff0 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f5568007ee0 tx=0x7f556800d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:44.594 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f556d7fa700 1 -- 172.21.15.159:0/1223711869 <== mon.0 v2:172.21.15.159:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5568017070 con 0x7f5570105a40 2023-03-27T10:22:44.595 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f556d7fa700 1 -- 172.21.15.159:0/1223711869 <== mon.0 v2:172.21.15.159:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f556800f040 con 0x7f5570105a40 2023-03-27T10:22:44.595 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f55763b7700 1 -- 172.21.15.159:0/1223711869 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f557010b640 con 0x7f5570105a40 2023-03-27T10:22:44.595 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f556d7fa700 1 -- 172.21.15.159:0/1223711869 <== mon.0 v2:172.21.15.159:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5568005740 con 0x7f5570105a40 2023-03-27T10:22:44.595 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.589+0000 7f55763b7700 1 -- 172.21.15.159:0/1223711869 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f557010b4b0 con 0x7f5570105a40 2023-03-27T10:22:44.595 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.590+0000 7f55763b7700 1 -- 172.21.15.159:0/1223711869 --> [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f55700fbd50 con 0x7f5570105a40 2023-03-27T10:22:44.596 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.591+0000 7f556d7fa700 1 -- 172.21.15.159:0/1223711869 <== mon.0 v2:172.21.15.159:3300/0 4 ==== mgrmap(e 16) v1 ==== 129858+0+0 (secure 0 0 0) 0x7f55680058a0 con 0x7f5570105a40 2023-03-27T10:22:44.596 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.592+0000 7f556d7fa700 1 --2- 172.21.15.159:0/1223711869 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f555809b3a0 0x7f555809d850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-03-27T10:22:44.597 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.592+0000 7f556ffff700 1 --2- 172.21.15.159:0/1223711869 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f555809b3a0 0x7f555809d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-03-27T10:22:44.597 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.592+0000 7f556d7fa700 1 -- 172.21.15.159:0/1223711869 <== mon.0 v2:172.21.15.159:3300/0 5 ==== osd_map(89..89 src has 1..89) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f55680c4b20 con 0x7f5570105a40 2023-03-27T10:22:44.597 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.592+0000 7f556ffff700 1 --2- 172.21.15.159:0/1223711869 >> [v2:172.21.15.159:6800/3290710483,v1:172.21.15.159:6801/3290710483] conn(0x7f555809b3a0 0x7f555809d850 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f5560006010 tx=0x7f556000b540 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-03-27T10:22:44.597 INFO:teuthology.orchestra.run.smithi159.stderr:2023-03-27T10:22:44.596+0000 7f556d7fa700 1 -- 172.21.15.159:0/1223711869 <== mon.0 v2:172.21.15.159:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+174404 (secure 0 0 0) 0x7f5568090250 con 0x7f5570105a40 2023-03-27T10:22:44.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:44 smithi171 ceph-mon[94808]: pgmap v267: 129 pgs: 129 active+clean; 581 KiB data, 78 MiB used, 1.0 TiB / 1.0 TiB avail; 383 B/s wr, 0 op/s 2023-03-27T10:22:44.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:44 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/2949905518' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-03-27T10:22:44.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:44 smithi171 ceph-mon[94808]: from='client.? 172.21.15.159:0/3660026272' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-03-27T10:22:44.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:44 smithi161 ceph-mon[94333]: pgmap v267: 129 pgs: 129 active+clean; 581 KiB data, 78 MiB used, 1.0 TiB / 1.0 TiB avail; 383 B/s wr, 0 op/s 2023-03-27T10:22:44.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:44 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/2949905518' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-03-27T10:22:44.834 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:44 smithi161 ceph-mon[94333]: from='client.? 172.21.15.159:0/3660026272' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-03-27T10:22:44.864 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:44 smithi159 ceph-mon[93039]: pgmap v267: 129 pgs: 129 active+clean; 581 KiB data, 78 MiB used, 1.0 TiB / 1.0 TiB avail; 383 B/s wr, 0 op/s 2023-03-27T10:22:44.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:44 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/2949905518' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-03-27T10:22:44.865 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:44 smithi159 ceph-mon[93039]: from='client.? 172.21.15.159:0/3660026272' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-03-27T10:22:45.007 INFO:teuthology.orchestra.run.smithi159.stderr:no valid command found; 10 closest matches: 2023-03-27T10:22:45.007 INFO:teuthology.orchestra.run.smithi159.stderr:pg stat 2023-03-27T10:22:45.008 INFO:teuthology.orchestra.run.smithi159.stderr:pg getmap 2023-03-27T10:22:45.008 INFO:teuthology.orchestra.run.smithi159.stderr:pg dump [...] 2023-03-27T10:22:45.008 INFO:teuthology.orchestra.run.smithi159.stderr:pg dump_json [...] 2023-03-27T10:22:45.008 INFO:teuthology.orchestra.run.smithi159.stderr:pg dump_pools_json 2023-03-27T10:22:45.008 INFO:teuthology.orchestra.run.smithi159.stderr:pg ls-by-pool [...] 2023-03-27T10:22:45.008 INFO:teuthology.orchestra.run.smithi159.stderr:pg ls-by-primary [] [...] 2023-03-27T10:22:45.008 INFO:teuthology.orchestra.run.smithi159.stderr:pg ls-by-osd [] [...] 2023-03-27T10:22:45.009 INFO:teuthology.orchestra.run.smithi159.stderr:pg ls [] [...] 2023-03-27T10:22:45.009 INFO:teuthology.orchestra.run.smithi159.stderr:pg dump_stuck [...] [] 2023-03-27T10:22:45.009 INFO:teuthology.orchestra.run.smithi159.stderr:Error EINVAL: invalid command 2023-03-27T10:22:45.010 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-03-27T10:22:45.011 ERROR:teuthology.contextutil:Saw exception from nested tasks Traceback (most recent call last): File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/mount.py", line 192, in is_blocked output = self.fs.get_ceph_cmd_stdout('osd blocklist ls') File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/filesystem.py", line 248, in get_ceph_cmd_stdout return self.mon_manager.run_cluster_cmd(**kwargs).stdout.getvalue() File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/ceph_manager.py", line 1579, in run_cluster_cmd return self.controller.run(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 525, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi159 with status 22: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph 'osd blocklist ls'" During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/contextutil.py", line 31, in nested vars.append(enter()) File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephadm.py", line 974, in cephfs_setup fs = Filesystem(ctx, fs_config=temp, name=name, create=True) File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/filesystem.py", line 538, in __init__ self.create() File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/filesystem.py", line 723, in create self.run_client_payload(f"setfattr -n ceph.dir.layout.pool -v {ec_data_pool_name} . && getfattr -n ceph.dir.layout .") File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/filesystem.py", line 798, in run_client_payload m.umount_wait(require_clean=True) File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/fuse_mount.py", line 389, in umount_wait if self.is_blocked(): File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/mount.py", line 195, in is_blocked output = self.fs.get_ceph_cmd_stdout('osd blacklist ls') File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/filesystem.py", line 248, in get_ceph_cmd_stdout return self.mon_manager.run_cluster_cmd(**kwargs).stdout.getvalue() File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/ceph_manager.py", line 1579, in run_cluster_cmd return self.controller.run(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 525, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi159 with status 22: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph 'osd blacklist ls'" 2023-03-27T10:22:45.012 DEBUG:teuthology.orchestra.run.smithi159:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-03-27T10:22:45.038 DEBUG:teuthology.orchestra.run.smithi161:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-03-27T10:22:45.077 DEBUG:teuthology.orchestra.run.smithi171:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-03-27T10:22:45.115 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-03-27T10:22:45.115 DEBUG:teuthology.orchestra.run.smithi159:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-03-27T10:22:45.132 DEBUG:teuthology.orchestra.run.smithi161:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-03-27T10:22:45.151 DEBUG:teuthology.orchestra.run.smithi171:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-03-27T10:22:45.173 INFO:tasks.cephadm:Stopping all daemons... 2023-03-27T10:22:45.173 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-03-27T10:22:45.174 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mon.a 2023-03-27T10:22:45.364 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:45 smithi159 systemd[1]: Stopping Ceph mon.a for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:45.795 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:45 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mon-a[93014]: 2023-03-27T10:22:45.494+0000 7f3c66372700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:22:45.796 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:45 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mon-a[93014]: 2023-03-27T10:22:45.494+0000 7f3c66372700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-03-27T10:22:46.088 INFO:journalctl@ceph.mon.a.smithi159.stdout:Mar 27 10:22:45 smithi159 bash[130756]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mon-a 2023-03-27T10:22:46.095 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mon.a.service' 2023-03-27T10:22:46.142 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:22:46.143 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-03-27T10:22:46.143 INFO:tasks.cephadm.mon.c:Stopping mon.b... 2023-03-27T10:22:46.143 DEBUG:teuthology.orchestra.run.smithi161:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mon.b 2023-03-27T10:22:46.510 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:46 smithi161 systemd[1]: Stopping Ceph mon.b for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:46.815 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:46 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mon-b[94309]: 2023-03-27T10:22:46.508+0000 7f68b0d16700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:22:46.815 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:46 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mon-b[94309]: 2023-03-27T10:22:46.508+0000 7f68b0d16700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2023-03-27T10:22:47.084 INFO:journalctl@ceph.mon.b.smithi161.stdout:Mar 27 10:22:46 smithi161 bash[120618]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mon-b 2023-03-27T10:22:47.315 DEBUG:teuthology.orchestra.run.smithi161:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mon.b.service' 2023-03-27T10:22:47.389 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:22:47.390 INFO:tasks.cephadm.mon.c:Stopped mon.b 2023-03-27T10:22:47.390 INFO:tasks.cephadm.mon.c:Stopping mon.c... 2023-03-27T10:22:47.391 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mon.c 2023-03-27T10:22:47.726 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:47 smithi171 systemd[1]: Stopping Ceph mon.c for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:48.016 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:47 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mon-c[94784]: 2023-03-27T10:22:47.751+0000 7f295a479700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:22:48.016 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:47 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mon-c[94784]: 2023-03-27T10:22:47.751+0000 7f295a479700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2023-03-27T10:22:48.337 INFO:journalctl@ceph.mon.c.smithi171.stdout:Mar 27 10:22:48 smithi171 bash[122249]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mon-c 2023-03-27T10:22:48.344 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mon.c.service' 2023-03-27T10:22:48.395 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:22:48.396 INFO:tasks.cephadm.mon.c:Stopped mon.c 2023-03-27T10:22:48.396 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-03-27T10:22:48.397 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mgr.x 2023-03-27T10:22:48.748 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:22:48 smithi159 systemd[1]: Stopping Ceph mgr.x for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:49.020 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:22:48 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x[93289]: 2023-03-27T10:22:48.747+0000 7fc9e83df700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.x -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:22:49.281 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Mar 27 10:22:49 smithi159 bash[131272]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-x 2023-03-27T10:22:49.288 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mgr.x.service' 2023-03-27T10:22:49.328 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:22:49.329 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-03-27T10:22:49.329 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-03-27T10:22:49.330 DEBUG:teuthology.orchestra.run.smithi161:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mgr.y 2023-03-27T10:22:49.758 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:22:49 smithi161 systemd[1]: Stopping Ceph mgr.y for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:50.032 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:22:49 smithi161 bash[121001]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-y 2023-03-27T10:22:50.214 INFO:journalctl@ceph.mgr.y.smithi161.stdout:Mar 27 10:22:50 smithi161 systemd[1]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-03-27T10:22:50.217 DEBUG:teuthology.orchestra.run.smithi161:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mgr.y.service' 2023-03-27T10:22:50.255 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:22:50.256 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-03-27T10:22:50.257 INFO:tasks.cephadm.mgr.z:Stopping mgr.z... 2023-03-27T10:22:50.257 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mgr.z 2023-03-27T10:22:50.678 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:22:50 smithi171 systemd[1]: Stopping Ceph mgr.z for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:50.966 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:22:50 smithi171 bash[122461]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mgr-z 2023-03-27T10:22:51.148 INFO:journalctl@ceph.mgr.z.smithi171.stdout:Mar 27 10:22:50 smithi171 systemd[1]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mgr.z.service: Main process exited, code=exited, status=143/n/a 2023-03-27T10:22:51.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mgr.z.service' 2023-03-27T10:22:51.189 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:22:51.190 INFO:tasks.cephadm.mgr.z:Stopped mgr.z 2023-03-27T10:22:51.190 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-03-27T10:22:51.190 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.0 2023-03-27T10:22:51.574 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 27 10:22:51 smithi159 systemd[1]: Stopping Ceph osd.0 for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:51.864 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 27 10:22:51 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-0[106031]: 2023-03-27T10:22:51.574+0000 7f116e2c4700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:22:51.865 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 27 10:22:51 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-0[106031]: 2023-03-27T10:22:51.574+0000 7f116e2c4700 -1 osd.0 89 *** Got signal Terminated *** 2023-03-27T10:22:51.865 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 27 10:22:51 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-0[106031]: 2023-03-27T10:22:51.574+0000 7f116e2c4700 -1 osd.0 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-03-27T10:22:57.428 INFO:journalctl@ceph.osd.0.smithi159.stdout:Mar 27 10:22:57 smithi159 bash[131413]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-0 2023-03-27T10:22:58.690 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.0.service' 2023-03-27T10:22:58.731 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:22:58.732 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-03-27T10:22:58.732 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-03-27T10:22:58.733 DEBUG:teuthology.orchestra.run.smithi161:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.1 2023-03-27T10:22:59.084 INFO:journalctl@ceph.osd.1.smithi161.stdout:Mar 27 10:22:58 smithi161 systemd[1]: Stopping Ceph osd.1 for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:22:59.584 INFO:journalctl@ceph.osd.1.smithi161.stdout:Mar 27 10:22:59 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-1[99296]: 2023-03-27T10:22:59.099+0000 7f13d29e4700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:22:59.584 INFO:journalctl@ceph.osd.1.smithi161.stdout:Mar 27 10:22:59 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-1[99296]: 2023-03-27T10:22:59.099+0000 7f13d29e4700 -1 osd.1 89 *** Got signal Terminated *** 2023-03-27T10:22:59.584 INFO:journalctl@ceph.osd.1.smithi161.stdout:Mar 27 10:22:59 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-1[99296]: 2023-03-27T10:22:59.099+0000 7f13d29e4700 -1 osd.1 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-03-27T10:23:04.773 INFO:journalctl@ceph.osd.1.smithi161.stdout:Mar 27 10:23:04 smithi161 bash[121146]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-1 2023-03-27T10:23:06.136 DEBUG:teuthology.orchestra.run.smithi161:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.1.service' 2023-03-27T10:23:06.178 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:23:06.179 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-03-27T10:23:06.180 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-03-27T10:23:06.181 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.2 2023-03-27T10:23:06.476 INFO:journalctl@ceph.osd.2.smithi171.stdout:Mar 27 10:23:06 smithi171 systemd[1]: Stopping Ceph osd.2 for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:23:06.976 INFO:journalctl@ceph.osd.2.smithi171.stdout:Mar 27 10:23:06 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-2[100593]: 2023-03-27T10:23:06.549+0000 7fe428040700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:23:06.976 INFO:journalctl@ceph.osd.2.smithi171.stdout:Mar 27 10:23:06 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-2[100593]: 2023-03-27T10:23:06.549+0000 7fe428040700 -1 osd.2 89 *** Got signal Terminated *** 2023-03-27T10:23:06.976 INFO:journalctl@ceph.osd.2.smithi171.stdout:Mar 27 10:23:06 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-2[100593]: 2023-03-27T10:23:06.549+0000 7fe428040700 -1 osd.2 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-03-27T10:23:12.220 INFO:journalctl@ceph.osd.2.smithi171.stdout:Mar 27 10:23:12 smithi171 bash[122607]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-2 2023-03-27T10:23:13.585 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.2.service' 2023-03-27T10:23:13.626 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:23:13.627 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-03-27T10:23:13.628 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-03-27T10:23:13.629 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.3 2023-03-27T10:23:14.030 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 27 10:23:13 smithi159 systemd[1]: Stopping Ceph osd.3 for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:23:14.364 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 27 10:23:14 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-3[111331]: 2023-03-27T10:23:14.030+0000 7f46b8ff3700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:23:14.365 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 27 10:23:14 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-3[111331]: 2023-03-27T10:23:14.030+0000 7f46b8ff3700 -1 osd.3 89 *** Got signal Terminated *** 2023-03-27T10:23:14.365 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 27 10:23:14 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-3[111331]: 2023-03-27T10:23:14.030+0000 7f46b8ff3700 -1 osd.3 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-03-27T10:23:20.115 INFO:journalctl@ceph.osd.3.smithi159.stdout:Mar 27 10:23:19 smithi159 bash[131682]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-3 2023-03-27T10:23:21.177 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.3.service' 2023-03-27T10:23:21.218 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:23:21.218 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-03-27T10:23:21.219 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-03-27T10:23:21.219 DEBUG:teuthology.orchestra.run.smithi161:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.4 2023-03-27T10:23:21.584 INFO:journalctl@ceph.osd.4.smithi161.stdout:Mar 27 10:23:21 smithi161 systemd[1]: Stopping Ceph osd.4 for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:23:22.084 INFO:journalctl@ceph.osd.4.smithi161.stdout:Mar 27 10:23:21 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-4[104810]: 2023-03-27T10:23:21.621+0000 7f5580589700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:23:22.084 INFO:journalctl@ceph.osd.4.smithi161.stdout:Mar 27 10:23:21 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-4[104810]: 2023-03-27T10:23:21.621+0000 7f5580589700 -1 osd.4 89 *** Got signal Terminated *** 2023-03-27T10:23:22.084 INFO:journalctl@ceph.osd.4.smithi161.stdout:Mar 27 10:23:21 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-4[104810]: 2023-03-27T10:23:21.621+0000 7f5580589700 -1 osd.4 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-03-27T10:23:27.583 INFO:journalctl@ceph.osd.4.smithi161.stdout:Mar 27 10:23:27 smithi161 bash[121419]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-4 2023-03-27T10:23:28.808 DEBUG:teuthology.orchestra.run.smithi161:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.4.service' 2023-03-27T10:23:28.849 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:23:28.849 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-03-27T10:23:28.850 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-03-27T10:23:28.850 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.5 2023-03-27T10:23:29.210 INFO:journalctl@ceph.osd.5.smithi171.stdout:Mar 27 10:23:28 smithi171 systemd[1]: Stopping Ceph osd.5 for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:23:29.476 INFO:journalctl@ceph.osd.5.smithi171.stdout:Mar 27 10:23:29 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-5[106084]: 2023-03-27T10:23:29.211+0000 7f0410411700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:23:29.476 INFO:journalctl@ceph.osd.5.smithi171.stdout:Mar 27 10:23:29 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-5[106084]: 2023-03-27T10:23:29.211+0000 7f0410411700 -1 osd.5 89 *** Got signal Terminated *** 2023-03-27T10:23:29.476 INFO:journalctl@ceph.osd.5.smithi171.stdout:Mar 27 10:23:29 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-5[106084]: 2023-03-27T10:23:29.211+0000 7f0410411700 -1 osd.5 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-03-27T10:23:34.976 INFO:journalctl@ceph.osd.5.smithi171.stdout:Mar 27 10:23:34 smithi171 bash[122877]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-5 2023-03-27T10:23:36.356 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.5.service' 2023-03-27T10:23:36.396 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:23:36.397 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-03-27T10:23:36.398 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-03-27T10:23:36.398 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.6 2023-03-27T10:23:36.748 INFO:journalctl@ceph.osd.6.smithi159.stdout:Mar 27 10:23:36 smithi159 systemd[1]: Stopping Ceph osd.6 for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:23:37.114 INFO:journalctl@ceph.osd.6.smithi159.stdout:Mar 27 10:23:36 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-6[116815]: 2023-03-27T10:23:36.748+0000 7f8eb4584700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:23:37.115 INFO:journalctl@ceph.osd.6.smithi159.stdout:Mar 27 10:23:36 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-6[116815]: 2023-03-27T10:23:36.748+0000 7f8eb4584700 -1 osd.6 89 *** Got signal Terminated *** 2023-03-27T10:23:37.115 INFO:journalctl@ceph.osd.6.smithi159.stdout:Mar 27 10:23:36 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-6[116815]: 2023-03-27T10:23:36.748+0000 7f8eb4584700 -1 osd.6 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-03-27T10:23:42.865 INFO:journalctl@ceph.osd.6.smithi159.stdout:Mar 27 10:23:42 smithi159 bash[131961]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-6 2023-03-27T10:23:43.886 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.6.service' 2023-03-27T10:23:43.926 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:23:43.927 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-03-27T10:23:43.927 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-03-27T10:23:43.928 DEBUG:teuthology.orchestra.run.smithi161:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.7 2023-03-27T10:23:44.300 INFO:journalctl@ceph.osd.7.smithi161.stdout:Mar 27 10:23:43 smithi161 systemd[1]: Stopping Ceph osd.7 for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:23:44.584 INFO:journalctl@ceph.osd.7.smithi161.stdout:Mar 27 10:23:44 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-7[110246]: 2023-03-27T10:23:44.300+0000 7fcee2e9f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:23:44.584 INFO:journalctl@ceph.osd.7.smithi161.stdout:Mar 27 10:23:44 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-7[110246]: 2023-03-27T10:23:44.300+0000 7fcee2e9f700 -1 osd.7 89 *** Got signal Terminated *** 2023-03-27T10:23:44.584 INFO:journalctl@ceph.osd.7.smithi161.stdout:Mar 27 10:23:44 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-7[110246]: 2023-03-27T10:23:44.300+0000 7fcee2e9f700 -1 osd.7 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-03-27T10:23:50.083 INFO:journalctl@ceph.osd.7.smithi161.stdout:Mar 27 10:23:50 smithi161 bash[121698]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-7 2023-03-27T10:23:51.432 DEBUG:teuthology.orchestra.run.smithi161:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.7.service' 2023-03-27T10:23:51.471 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:23:51.472 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-03-27T10:23:51.473 INFO:tasks.cephadm.osd.8:Stopping osd.8... 2023-03-27T10:23:51.473 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.8 2023-03-27T10:23:51.840 INFO:journalctl@ceph.osd.8.smithi171.stdout:Mar 27 10:23:51 smithi171 systemd[1]: Stopping Ceph osd.8 for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:23:52.226 INFO:journalctl@ceph.osd.8.smithi171.stdout:Mar 27 10:23:51 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-8[111586]: 2023-03-27T10:23:51.840+0000 7f608b3a8700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.8 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:23:52.226 INFO:journalctl@ceph.osd.8.smithi171.stdout:Mar 27 10:23:51 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-8[111586]: 2023-03-27T10:23:51.840+0000 7f608b3a8700 -1 osd.8 89 *** Got signal Terminated *** 2023-03-27T10:23:52.226 INFO:journalctl@ceph.osd.8.smithi171.stdout:Mar 27 10:23:51 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-8[111586]: 2023-03-27T10:23:51.840+0000 7f608b3a8700 -1 osd.8 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-03-27T10:23:57.726 INFO:journalctl@ceph.osd.8.smithi171.stdout:Mar 27 10:23:57 smithi171 bash[123141]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-8 2023-03-27T10:23:58.882 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.8.service' 2023-03-27T10:23:58.923 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:23:58.924 INFO:tasks.cephadm.osd.8:Stopped osd.8 2023-03-27T10:23:58.924 INFO:tasks.cephadm.osd.9:Stopping osd.9... 2023-03-27T10:23:58.925 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.9 2023-03-27T10:23:59.312 INFO:journalctl@ceph.osd.9.smithi159.stdout:Mar 27 10:23:58 smithi159 systemd[1]: Stopping Ceph osd.9 for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:23:59.614 INFO:journalctl@ceph.osd.9.smithi159.stdout:Mar 27 10:23:59 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-9[122365]: 2023-03-27T10:23:59.313+0000 7fb9f723b700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.9 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:23:59.615 INFO:journalctl@ceph.osd.9.smithi159.stdout:Mar 27 10:23:59 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-9[122365]: 2023-03-27T10:23:59.313+0000 7fb9f723b700 -1 osd.9 89 *** Got signal Terminated *** 2023-03-27T10:23:59.615 INFO:journalctl@ceph.osd.9.smithi159.stdout:Mar 27 10:23:59 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-9[122365]: 2023-03-27T10:23:59.313+0000 7fb9f723b700 -1 osd.9 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-03-27T10:24:00.334 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:23:59 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:23:59.964+0000 7fecb9b62700 -1 osd.10 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.004247+0000 front 2023-03-27T10:22:55.004188+0000 (oldest deadline 2023-03-27T10:23:59.704353+0000) 2023-03-27T10:24:00.614 INFO:journalctl@ceph.osd.9.smithi159.stdout:Mar 27 10:24:00 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-9[122365]: 2023-03-27T10:24:00.209+0000 7fb9f182d700 -1 osd.9 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:54.561150+0000 front 2023-03-27T10:22:54.561191+0000 (oldest deadline 2023-03-27T10:23:59.861277+0000) 2023-03-27T10:24:01.334 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:00 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:24:00.937+0000 7fecb9b62700 -1 osd.10 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.004247+0000 front 2023-03-27T10:22:55.004188+0000 (oldest deadline 2023-03-27T10:23:59.704353+0000) 2023-03-27T10:24:01.614 INFO:journalctl@ceph.osd.9.smithi159.stdout:Mar 27 10:24:01 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-9[122365]: 2023-03-27T10:24:01.187+0000 7fb9f182d700 -1 osd.9 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:54.561150+0000 front 2023-03-27T10:22:54.561191+0000 (oldest deadline 2023-03-27T10:23:59.861277+0000) 2023-03-27T10:24:01.976 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:01 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:01.715+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:02.333 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:01 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:24:01.936+0000 7fecb9b62700 -1 osd.10 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.004247+0000 front 2023-03-27T10:22:55.004188+0000 (oldest deadline 2023-03-27T10:23:59.704353+0000) 2023-03-27T10:24:02.614 INFO:journalctl@ceph.osd.9.smithi159.stdout:Mar 27 10:24:02 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-9[122365]: 2023-03-27T10:24:02.187+0000 7fb9f182d700 -1 osd.9 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:54.561150+0000 front 2023-03-27T10:22:54.561191+0000 (oldest deadline 2023-03-27T10:23:59.861277+0000) 2023-03-27T10:24:03.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:02 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:02.733+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:03.334 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:02 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:24:02.982+0000 7fecb9b62700 -1 osd.10 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.004247+0000 front 2023-03-27T10:22:55.004188+0000 (oldest deadline 2023-03-27T10:23:59.704353+0000) 2023-03-27T10:24:03.614 INFO:journalctl@ceph.osd.9.smithi159.stdout:Mar 27 10:24:03 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-9[122365]: 2023-03-27T10:24:03.235+0000 7fb9f182d700 -1 osd.9 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:54.561150+0000 front 2023-03-27T10:22:54.561191+0000 (oldest deadline 2023-03-27T10:23:59.861277+0000) 2023-03-27T10:24:03.976 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:03 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:03.705+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:04.334 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:03 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:24:03.992+0000 7fecb9b62700 -1 osd.10 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.004247+0000 front 2023-03-27T10:22:55.004188+0000 (oldest deadline 2023-03-27T10:23:59.704353+0000) 2023-03-27T10:24:04.615 INFO:journalctl@ceph.osd.9.smithi159.stdout:Mar 27 10:24:04 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-9[122365]: 2023-03-27T10:24:04.243+0000 7fb9f182d700 -1 osd.9 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:54.561150+0000 front 2023-03-27T10:22:54.561191+0000 (oldest deadline 2023-03-27T10:23:59.861277+0000) 2023-03-27T10:24:05.114 INFO:journalctl@ceph.osd.9.smithi159.stdout:Mar 27 10:24:04 smithi159 bash[132230]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-9 2023-03-27T10:24:05.225 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:04 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:04.743+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:05.333 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:04 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:24:04.972+0000 7fecb9b62700 -1 osd.10 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.004247+0000 front 2023-03-27T10:22:55.004188+0000 (oldest deadline 2023-03-27T10:23:59.704353+0000) 2023-03-27T10:24:06.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:05 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:05.728+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:06.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:05 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:05.728+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.161:6804 osd.1 since back 2023-03-27T10:23:00.974023+0000 front 2023-03-27T10:23:00.973919+0000 (oldest deadline 2023-03-27T10:24:05.073792+0000) 2023-03-27T10:24:06.320 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.9.service' 2023-03-27T10:24:06.333 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:05 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:24:05.946+0000 7fecb9b62700 -1 osd.10 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.004247+0000 front 2023-03-27T10:22:55.004188+0000 (oldest deadline 2023-03-27T10:23:59.704353+0000) 2023-03-27T10:24:06.360 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:24:06.360 INFO:tasks.cephadm.osd.9:Stopped osd.9 2023-03-27T10:24:06.361 INFO:tasks.cephadm.osd.10:Stopping osd.10... 2023-03-27T10:24:06.361 DEBUG:teuthology.orchestra.run.smithi161:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.10 2023-03-27T10:24:06.720 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:06 smithi161 systemd[1]: Stopping Ceph osd.10 for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:24:07.084 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:06 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:24:06.721+0000 7fecc1574700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.10 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:24:07.084 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:06 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:24:06.721+0000 7fecc1574700 -1 osd.10 89 *** Got signal Terminated *** 2023-03-27T10:24:07.084 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:06 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:24:06.721+0000 7fecc1574700 -1 osd.10 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-03-27T10:24:07.084 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:06 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:24:06.909+0000 7fecb9b62700 -1 osd.10 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.004247+0000 front 2023-03-27T10:22:55.004188+0000 (oldest deadline 2023-03-27T10:23:59.704353+0000) 2023-03-27T10:24:07.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:06 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:06.763+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:07.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:06 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:06.763+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.161:6804 osd.1 since back 2023-03-27T10:23:00.974023+0000 front 2023-03-27T10:23:00.973919+0000 (oldest deadline 2023-03-27T10:24:05.073792+0000) 2023-03-27T10:24:07.976 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:07 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:07.726+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:07.976 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:07 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:07.726+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.161:6804 osd.1 since back 2023-03-27T10:23:00.974023+0000 front 2023-03-27T10:23:00.973919+0000 (oldest deadline 2023-03-27T10:24:05.073792+0000) 2023-03-27T10:24:08.334 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:07 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:24:07.945+0000 7fecb9b62700 -1 osd.10 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.004247+0000 front 2023-03-27T10:22:55.004188+0000 (oldest deadline 2023-03-27T10:23:59.704353+0000) 2023-03-27T10:24:09.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:08 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:08.754+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:09.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:08 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:08.754+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.161:6804 osd.1 since back 2023-03-27T10:23:00.974023+0000 front 2023-03-27T10:23:00.973919+0000 (oldest deadline 2023-03-27T10:24:05.073792+0000) 2023-03-27T10:24:09.333 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:08 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:24:08.906+0000 7fecb9b62700 -1 osd.10 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.004247+0000 front 2023-03-27T10:22:55.004188+0000 (oldest deadline 2023-03-27T10:23:59.704353+0000) 2023-03-27T10:24:10.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:09 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:09.788+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:10.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:09 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:09.788+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.161:6804 osd.1 since back 2023-03-27T10:23:00.974023+0000 front 2023-03-27T10:23:00.973919+0000 (oldest deadline 2023-03-27T10:24:05.073792+0000) 2023-03-27T10:24:10.334 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:09 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:24:09.956+0000 7fecb9b62700 -1 osd.10 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.004247+0000 front 2023-03-27T10:22:55.004188+0000 (oldest deadline 2023-03-27T10:23:59.704353+0000) 2023-03-27T10:24:10.334 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:09 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:24:09.956+0000 7fecb9b62700 -1 osd.10 89 heartbeat_check: no reply from 172.21.15.161:6804 osd.1 since back 2023-03-27T10:23:03.105937+0000 front 2023-03-27T10:23:03.106068+0000 (oldest deadline 2023-03-27T10:24:09.006096+0000) 2023-03-27T10:24:11.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:10 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:10.813+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:11.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:10 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:10.813+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.161:6804 osd.1 since back 2023-03-27T10:23:00.974023+0000 front 2023-03-27T10:23:00.973919+0000 (oldest deadline 2023-03-27T10:24:05.073792+0000) 2023-03-27T10:24:11.334 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:10 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:24:10.974+0000 7fecb9b62700 -1 osd.10 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.004247+0000 front 2023-03-27T10:22:55.004188+0000 (oldest deadline 2023-03-27T10:23:59.704353+0000) 2023-03-27T10:24:11.334 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:10 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10[115660]: 2023-03-27T10:24:10.974+0000 7fecb9b62700 -1 osd.10 89 heartbeat_check: no reply from 172.21.15.161:6804 osd.1 since back 2023-03-27T10:23:03.105937+0000 front 2023-03-27T10:23:03.106068+0000 (oldest deadline 2023-03-27T10:24:09.006096+0000) 2023-03-27T10:24:12.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:11 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:11.785+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:12.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:11 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:11.785+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.161:6804 osd.1 since back 2023-03-27T10:23:00.974023+0000 front 2023-03-27T10:23:00.973919+0000 (oldest deadline 2023-03-27T10:24:05.073792+0000) 2023-03-27T10:24:12.584 INFO:journalctl@ceph.osd.10.smithi161.stdout:Mar 27 10:24:12 smithi161 bash[121971]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-10 2023-03-27T10:24:13.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:12 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:12.737+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:13.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:12 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:12.737+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.161:6804 osd.1 since back 2023-03-27T10:23:00.974023+0000 front 2023-03-27T10:23:00.973919+0000 (oldest deadline 2023-03-27T10:24:05.073792+0000) 2023-03-27T10:24:13.658 DEBUG:teuthology.orchestra.run.smithi161:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.10.service' 2023-03-27T10:24:13.699 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:24:13.699 INFO:tasks.cephadm.osd.10:Stopped osd.10 2023-03-27T10:24:13.700 INFO:tasks.cephadm.osd.11:Stopping osd.11... 2023-03-27T10:24:13.700 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.11 2023-03-27T10:24:14.079 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:13 smithi171 systemd[1]: Stopping Ceph osd.11 for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:24:14.080 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:13 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:13.779+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:14.080 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:13 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:13.779+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.161:6804 osd.1 since back 2023-03-27T10:23:00.974023+0000 front 2023-03-27T10:23:00.973919+0000 (oldest deadline 2023-03-27T10:24:05.073792+0000) 2023-03-27T10:24:14.080 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:14 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:14.080+0000 7f4049b34700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.11 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:24:14.080 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:14 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:14.080+0000 7f4049b34700 -1 osd.11 89 *** Got signal Terminated *** 2023-03-27T10:24:14.476 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:14 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:14.080+0000 7f4049b34700 -1 osd.11 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-03-27T10:24:15.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:14 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:14.799+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:15.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:14 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:14.799+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.161:6804 osd.1 since back 2023-03-27T10:23:00.974023+0000 front 2023-03-27T10:23:00.973919+0000 (oldest deadline 2023-03-27T10:24:05.073792+0000) 2023-03-27T10:24:16.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:15 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:15.845+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:16.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:15 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:15.845+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.161:6804 osd.1 since back 2023-03-27T10:23:00.974023+0000 front 2023-03-27T10:23:00.973919+0000 (oldest deadline 2023-03-27T10:24:05.073792+0000) 2023-03-27T10:24:16.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:15 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:15.845+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.171:6804 osd.2 since back 2023-03-27T10:23:11.375760+0000 front 2023-03-27T10:23:11.375849+0000 (oldest deadline 2023-03-27T10:24:15.475974+0000) 2023-03-27T10:24:17.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:16 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:16.853+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:17.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:16 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:16.853+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.161:6804 osd.1 since back 2023-03-27T10:23:00.974023+0000 front 2023-03-27T10:23:00.973919+0000 (oldest deadline 2023-03-27T10:24:05.073792+0000) 2023-03-27T10:24:17.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:16 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:16.853+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.171:6804 osd.2 since back 2023-03-27T10:23:11.375760+0000 front 2023-03-27T10:23:11.375849+0000 (oldest deadline 2023-03-27T10:24:15.475974+0000) 2023-03-27T10:24:18.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:17 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:17.880+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:18.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:17 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:17.880+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.161:6804 osd.1 since back 2023-03-27T10:23:00.974023+0000 front 2023-03-27T10:23:00.973919+0000 (oldest deadline 2023-03-27T10:24:05.073792+0000) 2023-03-27T10:24:18.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:17 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:17.880+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.171:6804 osd.2 since back 2023-03-27T10:23:11.375760+0000 front 2023-03-27T10:23:11.375849+0000 (oldest deadline 2023-03-27T10:24:15.475974+0000) 2023-03-27T10:24:19.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:18 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:18.920+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.159:6806 osd.0 since back 2023-03-27T10:22:55.073695+0000 front 2023-03-27T10:22:55.073364+0000 (oldest deadline 2023-03-27T10:24:00.973371+0000) 2023-03-27T10:24:19.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:18 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:18.920+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.161:6804 osd.1 since back 2023-03-27T10:23:00.974023+0000 front 2023-03-27T10:23:00.973919+0000 (oldest deadline 2023-03-27T10:24:05.073792+0000) 2023-03-27T10:24:19.226 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:18 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11[117075]: 2023-03-27T10:24:18.920+0000 7f4042923700 -1 osd.11 89 heartbeat_check: no reply from 172.21.15.171:6804 osd.2 since back 2023-03-27T10:23:11.375760+0000 front 2023-03-27T10:23:11.375849+0000 (oldest deadline 2023-03-27T10:24:15.475974+0000) 2023-03-27T10:24:19.976 INFO:journalctl@ceph.osd.11.smithi171.stdout:Mar 27 10:24:19 smithi171 bash[123407]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-osd-11 2023-03-27T10:24:21.001 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@osd.11.service' 2023-03-27T10:24:21.042 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:24:21.043 INFO:tasks.cephadm.osd.11:Stopped osd.11 2023-03-27T10:24:21.043 INFO:tasks.cephadm.mds.a:Stopping mds.a... 2023-03-27T10:24:21.044 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.a 2023-03-27T10:24:21.364 INFO:journalctl@ceph.mds.a.smithi159.stdout:Mar 27 10:24:21 smithi159 systemd[1]: Stopping Ceph mds.a for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:24:21.864 INFO:journalctl@ceph.mds.a.smithi159.stdout:Mar 27 10:24:21 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-a[127646]: 2023-03-27T10:24:21.407+0000 7fdec4411700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:24:21.865 INFO:journalctl@ceph.mds.a.smithi159.stdout:Mar 27 10:24:21 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-a[127646]: 2023-03-27T10:24:21.407+0000 7fdec4411700 -1 mds.a *** got signal Terminated *** 2023-03-27T10:24:23.765 INFO:journalctl@ceph.mds.a.smithi159.stdout:Mar 27 10:24:23 smithi159 bash[132501]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-a 2023-03-27T10:24:23.955 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.a.service' 2023-03-27T10:24:23.993 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:24:23.994 INFO:tasks.cephadm.mds.a:Stopped mds.a 2023-03-27T10:24:23.994 INFO:tasks.cephadm.mds.d:Stopping mds.d... 2023-03-27T10:24:23.995 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.d 2023-03-27T10:24:24.364 INFO:journalctl@ceph.mds.d.smithi159.stdout:Mar 27 10:24:24 smithi159 systemd[1]: Stopping Ceph mds.d for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:24:24.864 INFO:journalctl@ceph.mds.d.smithi159.stdout:Mar 27 10:24:24 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-d[126644]: 2023-03-27T10:24:24.366+0000 7f363a5d6700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.d -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:24:24.865 INFO:journalctl@ceph.mds.d.smithi159.stdout:Mar 27 10:24:24 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-d[126644]: 2023-03-27T10:24:24.366+0000 7f363a5d6700 -1 mds.d *** got signal Terminated *** 2023-03-27T10:24:28.750 INFO:journalctl@ceph.mds.d.smithi159.stdout:Mar 27 10:24:28 smithi159 bash[132639]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-d 2023-03-27T10:24:28.756 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.d.service' 2023-03-27T10:24:28.792 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:24:28.792 INFO:tasks.cephadm.mds.d:Stopped mds.d 2023-03-27T10:24:28.793 INFO:tasks.cephadm.mds.g:Stopping mds.g... 2023-03-27T10:24:28.793 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.g 2023-03-27T10:24:29.114 INFO:journalctl@ceph.mds.g.smithi159.stdout:Mar 27 10:24:28 smithi159 systemd[1]: Stopping Ceph mds.g for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:24:29.614 INFO:journalctl@ceph.mds.g.smithi159.stdout:Mar 27 10:24:29 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-g[127262]: 2023-03-27T10:24:29.165+0000 7fb406ae1700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.g -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:24:29.615 INFO:journalctl@ceph.mds.g.smithi159.stdout:Mar 27 10:24:29 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-g[127262]: 2023-03-27T10:24:29.165+0000 7fb406ae1700 -1 mds.g *** got signal Terminated *** 2023-03-27T10:24:33.614 INFO:journalctl@ceph.mds.g.smithi159.stdout:Mar 27 10:24:33 smithi159 bash[132792]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-g 2023-03-27T10:24:33.718 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.g.service' 2023-03-27T10:24:33.752 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:24:33.753 INFO:tasks.cephadm.mds.g:Stopped mds.g 2023-03-27T10:24:33.754 INFO:tasks.cephadm.mds.j:Stopping mds.j... 2023-03-27T10:24:33.754 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.j 2023-03-27T10:24:34.114 INFO:journalctl@ceph.mds.j.smithi159.stdout:Mar 27 10:24:33 smithi159 systemd[1]: Stopping Ceph mds.j for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:24:34.615 INFO:journalctl@ceph.mds.j.smithi159.stdout:Mar 27 10:24:34 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-j[126258]: 2023-03-27T10:24:34.162+0000 7fb0740aa700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.j -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:24:34.615 INFO:journalctl@ceph.mds.j.smithi159.stdout:Mar 27 10:24:34 smithi159 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-j[126258]: 2023-03-27T10:24:34.162+0000 7fb0740aa700 -1 mds.j *** got signal Terminated *** 2023-03-27T10:24:38.304 INFO:journalctl@ceph.mds.j.smithi159.stdout:Mar 27 10:24:38 smithi159 bash[132936]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-j 2023-03-27T10:24:38.504 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.j.service' 2023-03-27T10:24:38.548 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:24:38.549 INFO:tasks.cephadm.mds.j:Stopped mds.j 2023-03-27T10:24:38.550 INFO:tasks.cephadm.mds.b:Stopping mds.b... 2023-03-27T10:24:38.550 DEBUG:teuthology.orchestra.run.smithi161:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.b 2023-03-27T10:24:38.834 INFO:journalctl@ceph.mds.b.smithi161.stdout:Mar 27 10:24:38 smithi161 systemd[1]: Stopping Ceph mds.b for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:24:39.334 INFO:journalctl@ceph.mds.b.smithi161.stdout:Mar 27 10:24:38 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-b[119432]: 2023-03-27T10:24:38.908+0000 7f3366a8c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:24:39.334 INFO:journalctl@ceph.mds.b.smithi161.stdout:Mar 27 10:24:38 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-b[119432]: 2023-03-27T10:24:38.908+0000 7f3366a8c700 -1 mds.b *** got signal Terminated *** 2023-03-27T10:24:47.584 INFO:journalctl@ceph.mds.b.smithi161.stdout:Mar 27 10:24:47 smithi161 bash[122234]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-b 2023-03-27T10:24:47.796 DEBUG:teuthology.orchestra.run.smithi161:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.b.service' 2023-03-27T10:24:47.832 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:24:47.833 INFO:tasks.cephadm.mds.b:Stopped mds.b 2023-03-27T10:24:47.833 INFO:tasks.cephadm.mds.e:Stopping mds.e... 2023-03-27T10:24:47.834 DEBUG:teuthology.orchestra.run.smithi161:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.e 2023-03-27T10:24:48.083 INFO:journalctl@ceph.mds.e.smithi161.stdout:Mar 27 10:24:47 smithi161 systemd[1]: Stopping Ceph mds.e for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:24:48.584 INFO:journalctl@ceph.mds.e.smithi161.stdout:Mar 27 10:24:48 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-e[118749]: 2023-03-27T10:24:48.207+0000 7f56ba793700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.e -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:24:48.584 INFO:journalctl@ceph.mds.e.smithi161.stdout:Mar 27 10:24:48 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-e[118749]: 2023-03-27T10:24:48.207+0000 7f56ba793700 -1 mds.e *** got signal Terminated *** 2023-03-27T10:24:51.084 INFO:journalctl@ceph.mds.e.smithi161.stdout:Mar 27 10:24:50 smithi161 bash[122374]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-e 2023-03-27T10:24:51.091 DEBUG:teuthology.orchestra.run.smithi161:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.e.service' 2023-03-27T10:24:51.124 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:24:51.125 INFO:tasks.cephadm.mds.e:Stopped mds.e 2023-03-27T10:24:51.125 INFO:tasks.cephadm.mds.h:Stopping mds.h... 2023-03-27T10:24:51.126 DEBUG:teuthology.orchestra.run.smithi161:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.h 2023-03-27T10:24:51.521 INFO:journalctl@ceph.mds.h.smithi161.stdout:Mar 27 10:24:51 smithi161 systemd[1]: Stopping Ceph mds.h for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:24:51.834 INFO:journalctl@ceph.mds.h.smithi161.stdout:Mar 27 10:24:51 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-h[119090]: 2023-03-27T10:24:51.522+0000 7f576851d700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.h -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:24:51.834 INFO:journalctl@ceph.mds.h.smithi161.stdout:Mar 27 10:24:51 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-h[119090]: 2023-03-27T10:24:51.522+0000 7f576851d700 -1 mds.h *** got signal Terminated *** 2023-03-27T10:24:53.491 INFO:journalctl@ceph.mds.h.smithi161.stdout:Mar 27 10:24:53 smithi161 bash[122521]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-h 2023-03-27T10:24:53.497 DEBUG:teuthology.orchestra.run.smithi161:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.h.service' 2023-03-27T10:24:53.532 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:24:53.533 INFO:tasks.cephadm.mds.h:Stopped mds.h 2023-03-27T10:24:53.533 INFO:tasks.cephadm.mds.k:Stopping mds.k... 2023-03-27T10:24:53.534 DEBUG:teuthology.orchestra.run.smithi161:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.k 2023-03-27T10:24:53.833 INFO:journalctl@ceph.mds.k.smithi161.stdout:Mar 27 10:24:53 smithi161 systemd[1]: Stopping Ceph mds.k for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:24:54.333 INFO:journalctl@ceph.mds.k.smithi161.stdout:Mar 27 10:24:53 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-k[118400]: 2023-03-27T10:24:53.938+0000 7ffa83007700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.k -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:24:54.334 INFO:journalctl@ceph.mds.k.smithi161.stdout:Mar 27 10:24:53 smithi161 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-k[118400]: 2023-03-27T10:24:53.938+0000 7ffa83007700 -1 mds.k *** got signal Terminated *** 2023-03-27T10:24:56.230 INFO:journalctl@ceph.mds.k.smithi161.stdout:Mar 27 10:24:56 smithi161 bash[122671]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-k 2023-03-27T10:24:56.372 DEBUG:teuthology.orchestra.run.smithi161:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.k.service' 2023-03-27T10:24:56.404 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:24:56.405 INFO:tasks.cephadm.mds.k:Stopped mds.k 2023-03-27T10:24:56.406 INFO:tasks.cephadm.mds.c:Stopping mds.c... 2023-03-27T10:24:56.406 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.c 2023-03-27T10:24:56.726 INFO:journalctl@ceph.mds.c.smithi171.stdout:Mar 27 10:24:56 smithi171 systemd[1]: Stopping Ceph mds.c for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:24:57.226 INFO:journalctl@ceph.mds.c.smithi171.stdout:Mar 27 10:24:56 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-c[120572]: 2023-03-27T10:24:56.781+0000 7f79caca9700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:24:57.226 INFO:journalctl@ceph.mds.c.smithi171.stdout:Mar 27 10:24:56 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-c[120572]: 2023-03-27T10:24:56.781+0000 7f79caca9700 -1 mds.c *** got signal Terminated *** 2023-03-27T10:24:59.038 INFO:journalctl@ceph.mds.c.smithi171.stdout:Mar 27 10:24:58 smithi171 bash[123674]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-c 2023-03-27T10:24:59.232 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.c.service' 2023-03-27T10:24:59.269 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:24:59.270 INFO:tasks.cephadm.mds.c:Stopped mds.c 2023-03-27T10:24:59.271 INFO:tasks.cephadm.mds.f:Stopping mds.f... 2023-03-27T10:24:59.271 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.f 2023-03-27T10:24:59.662 INFO:journalctl@ceph.mds.f.smithi171.stdout:Mar 27 10:24:59 smithi171 systemd[1]: Stopping Ceph mds.f for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:24:59.975 INFO:journalctl@ceph.mds.f.smithi171.stdout:Mar 27 10:24:59 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-f[120230]: 2023-03-27T10:24:59.663+0000 7fc0c786c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.f -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:24:59.976 INFO:journalctl@ceph.mds.f.smithi171.stdout:Mar 27 10:24:59 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-f[120230]: 2023-03-27T10:24:59.663+0000 7fc0c786c700 -1 mds.f *** got signal Terminated *** 2023-03-27T10:25:01.191 INFO:journalctl@ceph.mds.f.smithi171.stdout:Mar 27 10:25:00 smithi171 bash[123813]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-f 2023-03-27T10:25:01.197 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.f.service' 2023-03-27T10:25:01.231 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:25:01.232 INFO:tasks.cephadm.mds.f:Stopped mds.f 2023-03-27T10:25:01.232 INFO:tasks.cephadm.mds.i:Stopping mds.i... 2023-03-27T10:25:01.233 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.i 2023-03-27T10:25:01.476 INFO:journalctl@ceph.mds.i.smithi171.stdout:Mar 27 10:25:01 smithi171 systemd[1]: Stopping Ceph mds.i for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:25:01.976 INFO:journalctl@ceph.mds.i.smithi171.stdout:Mar 27 10:25:01 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-i[120913]: 2023-03-27T10:25:01.612+0000 7fa6591f2700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.i -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:25:01.976 INFO:journalctl@ceph.mds.i.smithi171.stdout:Mar 27 10:25:01 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-i[120913]: 2023-03-27T10:25:01.612+0000 7fa6591f2700 -1 mds.i *** got signal Terminated *** 2023-03-27T10:25:03.370 INFO:journalctl@ceph.mds.i.smithi171.stdout:Mar 27 10:25:03 smithi171 bash[123962]: ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-i 2023-03-27T10:25:03.558 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.i.service' 2023-03-27T10:25:03.592 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:25:03.593 INFO:tasks.cephadm.mds.i:Stopped mds.i 2023-03-27T10:25:03.593 INFO:tasks.cephadm.mds.l:Stopping mds.l... 2023-03-27T10:25:03.594 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.l 2023-03-27T10:25:03.970 INFO:journalctl@ceph.mds.l.smithi171.stdout:Mar 27 10:25:03 smithi171 systemd[1]: Stopping Ceph mds.l for 2c3cd140-cc88-11ed-9afe-001a4aab830c... 2023-03-27T10:25:04.225 INFO:journalctl@ceph.mds.l.smithi171.stdout:Mar 27 10:25:03 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-l[119890]: 2023-03-27T10:25:03.971+0000 7f08b397e700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.l -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-03-27T10:25:04.226 INFO:journalctl@ceph.mds.l.smithi171.stdout:Mar 27 10:25:03 smithi171 ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c-mds-l[119890]: 2023-03-27T10:25:03.971+0000 7f08b397e700 -1 mds.l *** got signal Terminated *** 2023-03-27T10:25:06.238 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2c3cd140-cc88-11ed-9afe-001a4aab830c@mds.l.service' 2023-03-27T10:25:06.271 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-03-27T10:25:06.272 INFO:tasks.cephadm.mds.l:Stopped mds.l 2023-03-27T10:25:06.272 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c --force --keep-logs 2023-03-27T10:25:08.246 DEBUG:teuthology.orchestra.run.smithi161:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c --force --keep-logs 2023-03-27T10:25:10.233 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c --force --keep-logs 2023-03-27T10:25:12.172 DEBUG:teuthology.orchestra.run.smithi159:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-03-27T10:25:12.209 DEBUG:teuthology.orchestra.run.smithi161:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-03-27T10:25:12.246 DEBUG:teuthology.orchestra.run.smithi171:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-03-27T10:25:12.276 INFO:tasks.cephadm:Archiving crash dumps... 2023-03-27T10:25:12.278 DEBUG:teuthology.misc:Transferring archived files from smithi159:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/crash to /home/teuthworker/archive/rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi/7221317/remote/smithi159/crash 2023-03-27T10:25:12.279 DEBUG:teuthology.orchestra.run.smithi159:> sudo tar cz -f - -C /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/crash -- . 2023-03-27T10:25:12.329 INFO:teuthology.orchestra.run.smithi159.stderr:tar: /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/crash: Cannot open: No such file or directory 2023-03-27T10:25:12.329 INFO:teuthology.orchestra.run.smithi159.stderr:tar: Error is not recoverable: exiting now 2023-03-27T10:25:12.331 DEBUG:teuthology.misc:Transferring archived files from smithi161:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/crash to /home/teuthworker/archive/rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi/7221317/remote/smithi161/crash 2023-03-27T10:25:12.332 DEBUG:teuthology.orchestra.run.smithi161:> sudo tar cz -f - -C /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/crash -- . 2023-03-27T10:25:12.375 INFO:teuthology.orchestra.run.smithi161.stderr:tar: /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/crash: Cannot open: No such file or directory 2023-03-27T10:25:12.376 INFO:teuthology.orchestra.run.smithi161.stderr:tar: Error is not recoverable: exiting now 2023-03-27T10:25:12.377 DEBUG:teuthology.misc:Transferring archived files from smithi171:/var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/crash to /home/teuthworker/archive/rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi/7221317/remote/smithi171/crash 2023-03-27T10:25:12.379 DEBUG:teuthology.orchestra.run.smithi171:> sudo tar cz -f - -C /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/crash -- . 2023-03-27T10:25:12.427 INFO:teuthology.orchestra.run.smithi171.stderr:tar: /var/lib/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/crash: Cannot open: No such file or directory 2023-03-27T10:25:12.427 INFO:teuthology.orchestra.run.smithi171.stderr:tar: Error is not recoverable: exiting now 2023-03-27T10:25:12.428 INFO:tasks.cephadm:Checking cluster log for badness... 2023-03-27T10:25:12.430 DEBUG:teuthology.orchestra.run.smithi159:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v 'overall HEALTH_' | egrep -v '\(FS_DEGRADED\)' | egrep -v '\(MDS_FAILED\)' | egrep -v '\(MDS_DEGRADED\)' | egrep -v '\(FS_WITH_FAILED_MDS\)' | egrep -v '\(MDS_DAMAGE\)' | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(FS_INLINE_DATA_DEPRECATED\)' | egrep -v 'overall HEALTH_' | egrep -v '\(OSD_DOWN\)' | egrep -v '\(OSD_' | egrep -v 'but it is still running' | egrep -v 'is not responding' | egrep -v SLOW_OPS | egrep -v 'slow request' | head -n 1 2023-03-27T10:25:12.466 INFO:teuthology.orchestra.run.smithi159.stderr:grep: /var/log/ceph/2c3cd140-cc88-11ed-9afe-001a4aab830c/ceph.log: No such file or directory 2023-03-27T10:25:12.468 INFO:tasks.cephadm:Compressing logs... 2023-03-27T10:25:12.468 DEBUG:teuthology.orchestra.run.smithi159:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-03-27T10:25:12.510 DEBUG:teuthology.orchestra.run.smithi161:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-03-27T10:25:12.513 DEBUG:teuthology.orchestra.run.smithi171:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-03-27T10:25:12.539 INFO:teuthology.orchestra.run.smithi159.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-03-27T10:25:12.540 INFO:teuthology.orchestra.run.smithi171.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-03-27T10:25:12.540 INFO:teuthology.orchestra.run.smithi161.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-03-27T10:25:13.639 INFO:tasks.cephadm:Archiving logs... 2023-03-27T10:25:13.640 DEBUG:teuthology.misc:Transferring archived files from smithi159:/var/log/ceph to /home/teuthworker/archive/rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi/7221317/remote/smithi159/log 2023-03-27T10:25:13.641 DEBUG:teuthology.orchestra.run.smithi159:> sudo tar cz -f - -C /var/log/ceph -- . 2023-03-27T10:25:13.953 DEBUG:teuthology.misc:Transferring archived files from smithi161:/var/log/ceph to /home/teuthworker/archive/rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi/7221317/remote/smithi161/log 2023-03-27T10:25:13.954 DEBUG:teuthology.orchestra.run.smithi161:> sudo tar cz -f - -C /var/log/ceph -- . 2023-03-27T10:25:14.182 DEBUG:teuthology.misc:Transferring archived files from smithi171:/var/log/ceph to /home/teuthworker/archive/rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi/7221317/remote/smithi171/log 2023-03-27T10:25:14.183 DEBUG:teuthology.orchestra.run.smithi171:> sudo tar cz -f - -C /var/log/ceph -- . 2023-03-27T10:25:14.394 INFO:tasks.cephadm:Removing cluster... 2023-03-27T10:25:14.394 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c --force 2023-03-27T10:25:15.228 DEBUG:teuthology.orchestra.run.smithi161:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c --force 2023-03-27T10:25:16.061 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2c3cd140-cc88-11ed-9afe-001a4aab830c --force 2023-03-27T10:25:16.908 INFO:tasks.cephadm:Removing cephadm ... 2023-03-27T10:25:16.909 DEBUG:teuthology.orchestra.run.smithi159:> rm -rf /home/ubuntu/cephtest/cephadm 2023-03-27T10:25:16.931 DEBUG:teuthology.orchestra.run.smithi161:> rm -rf /home/ubuntu/cephtest/cephadm 2023-03-27T10:25:16.953 DEBUG:teuthology.orchestra.run.smithi171:> rm -rf /home/ubuntu/cephtest/cephadm 2023-03-27T10:25:16.970 INFO:tasks.cephadm:Teardown complete 2023-03-27T10:25:16.971 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/mount.py", line 192, in is_blocked output = self.fs.get_ceph_cmd_stdout('osd blocklist ls') File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/filesystem.py", line 248, in get_ceph_cmd_stdout return self.mon_manager.run_cluster_cmd(**kwargs).stdout.getvalue() File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/ceph_manager.py", line 1579, in run_cluster_cmd return self.controller.run(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 525, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi159 with status 22: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph 'osd blocklist ls'" During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephadm.py", line 1637, in task with contextutil.nested( File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/contextutil.py", line 55, in nested raise exc[1] File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__ self.gen.throw(type, value, traceback) File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephadm.py", line 1552, in initialize_config yield File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/contextutil.py", line 31, in nested vars.append(enter()) File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephadm.py", line 974, in cephfs_setup fs = Filesystem(ctx, fs_config=temp, name=name, create=True) File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/filesystem.py", line 538, in __init__ self.create() File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/filesystem.py", line 723, in create self.run_client_payload(f"setfattr -n ceph.dir.layout.pool -v {ec_data_pool_name} . && getfattr -n ceph.dir.layout .") File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/filesystem.py", line 798, in run_client_payload m.umount_wait(require_clean=True) File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/fuse_mount.py", line 389, in umount_wait if self.is_blocked(): File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/mount.py", line 195, in is_blocked output = self.fs.get_ceph_cmd_stdout('osd blacklist ls') File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/filesystem.py", line 248, in get_ceph_cmd_stdout return self.mon_manager.run_cluster_cmd(**kwargs).stdout.getvalue() File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/ceph_manager.py", line 1579, in run_cluster_cmd return self.controller.run(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 525, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi159 with status 22: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph 'osd blacklist ls'" 2023-03-27T10:25:17.051 ERROR:teuthology.run_tasks: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=47e28ff1b3f648a883211cf5b55e7664 Traceback (most recent call last): File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/mount.py", line 192, in is_blocked output = self.fs.get_ceph_cmd_stdout('osd blocklist ls') File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/filesystem.py", line 248, in get_ceph_cmd_stdout return self.mon_manager.run_cluster_cmd(**kwargs).stdout.getvalue() File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/ceph_manager.py", line 1579, in run_cluster_cmd return self.controller.run(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 525, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi159 with status 22: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph 'osd blocklist ls'" During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephadm.py", line 1637, in task with contextutil.nested( File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/contextutil.py", line 55, in nested raise exc[1] File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__ self.gen.throw(type, value, traceback) File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephadm.py", line 1552, in initialize_config yield File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/contextutil.py", line 31, in nested vars.append(enter()) File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephadm.py", line 974, in cephfs_setup fs = Filesystem(ctx, fs_config=temp, name=name, create=True) File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/filesystem.py", line 538, in __init__ self.create() File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/filesystem.py", line 723, in create self.run_client_payload(f"setfattr -n ceph.dir.layout.pool -v {ec_data_pool_name} . && getfattr -n ceph.dir.layout .") File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/filesystem.py", line 798, in run_client_payload m.umount_wait(require_clean=True) File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/fuse_mount.py", line 389, in umount_wait if self.is_blocked(): File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/mount.py", line 195, in is_blocked output = self.fs.get_ceph_cmd_stdout('osd blacklist ls') File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/cephfs/filesystem.py", line 248, in get_ceph_cmd_stdout return self.mon_manager.run_cluster_cmd(**kwargs).stdout.getvalue() File "/home/teuthworker/src/github.com_rishabh-d-dave_ceph_342392241eddd9827618f3dbb8134df7441718a8/qa/tasks/ceph_manager.py", line 1579, in run_cluster_cmd return self.controller.run(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/remote.py", line 525, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_main/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi159 with status 22: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph 'osd blacklist ls'" 2023-03-27T10:25:17.058 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-03-27T10:25:17.070 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-03-27T10:25:17.104 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits... 2023-03-27T10:25:17.105 DEBUG:teuthology.orchestra.run.smithi159:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits 2023-03-27T10:25:17.128 DEBUG:teuthology.orchestra.run.smithi161:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits 2023-03-27T10:25:17.132 DEBUG:teuthology.orchestra.run.smithi171:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits 2023-03-27T10:25:17.241 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-03-27T10:25:17.250 INFO:teuthology.task.clock:Checking final clock skew... 2023-03-27T10:25:17.250 DEBUG:teuthology.orchestra.run.smithi159:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-03-27T10:25:17.253 DEBUG:teuthology.orchestra.run.smithi161:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-03-27T10:25:17.255 DEBUG:teuthology.orchestra.run.smithi171:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-03-27T10:25:17.269 INFO:teuthology.orchestra.run.smithi159.stderr:bash: ntpq: command not found 2023-03-27T10:25:17.271 INFO:teuthology.orchestra.run.smithi171.stderr:bash: ntpq: command not found 2023-03-27T10:25:17.272 INFO:teuthology.orchestra.run.smithi161.stderr:bash: ntpq: command not found 2023-03-27T10:25:17.273 INFO:teuthology.orchestra.run.smithi159.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-03-27T10:25:17.274 INFO:teuthology.orchestra.run.smithi159.stdout:=============================================================================== 2023-03-27T10:25:17.274 INFO:teuthology.orchestra.run.smithi159.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 23 +1352us[+1346us] +/- 85ms 2023-03-27T10:25:17.274 INFO:teuthology.orchestra.run.smithi159.stdout:^* hv02.front.sepia.ceph.com 3 6 377 20 +2567us[+2561us] +/- 66ms 2023-03-27T10:25:17.274 INFO:teuthology.orchestra.run.smithi159.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 23 -2820us[-2826us] +/- 51ms 2023-03-27T10:25:17.274 INFO:teuthology.orchestra.run.smithi159.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2023-03-27T10:25:17.276 INFO:teuthology.orchestra.run.smithi171.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-03-27T10:25:17.276 INFO:teuthology.orchestra.run.smithi171.stdout:=============================================================================== 2023-03-27T10:25:17.276 INFO:teuthology.orchestra.run.smithi171.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 20 +1377us[+1377us] +/- 85ms 2023-03-27T10:25:17.276 INFO:teuthology.orchestra.run.smithi171.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 21 +2588us[+2588us] +/- 66ms 2023-03-27T10:25:17.276 INFO:teuthology.orchestra.run.smithi171.stdout:^* hv03.front.sepia.ceph.com 3 6 377 24 -2790us[-2789us] +/- 51ms 2023-03-27T10:25:17.277 INFO:teuthology.orchestra.run.smithi171.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2023-03-27T10:25:17.278 INFO:teuthology.orchestra.run.smithi161.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-03-27T10:25:17.278 INFO:teuthology.orchestra.run.smithi161.stdout:=============================================================================== 2023-03-27T10:25:17.278 INFO:teuthology.orchestra.run.smithi161.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 22 +1386us[+1368us] +/- 85ms 2023-03-27T10:25:17.278 INFO:teuthology.orchestra.run.smithi161.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 21 +2619us[+2601us] +/- 66ms 2023-03-27T10:25:17.278 INFO:teuthology.orchestra.run.smithi161.stdout:^* hv03.front.sepia.ceph.com 3 6 377 23 -2771us[-2789us] +/- 51ms 2023-03-27T10:25:17.279 INFO:teuthology.orchestra.run.smithi161.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2023-03-27T10:25:17.279 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-03-27T10:25:17.287 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-03-27T10:25:17.288 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-03-27T10:25:17.313 DEBUG:teuthology.orchestra.run.smithi159:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-03-27T10:25:17.414 DEBUG:teuthology.orchestra.run.smithi161:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-03-27T10:25:17.512 DEBUG:teuthology.orchestra.run.smithi171:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-03-27T10:25:17.603 DEBUG:teuthology.orchestra.run.smithi159:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-03-27T10:25:17.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-03-27T10:25:17.637 DEBUG:teuthology.orchestra.run.smithi161:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-03-27T10:25:17.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-03-27T10:25:17.672 DEBUG:teuthology.orchestra.run.smithi171:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-03-27T10:25:17.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-03-27T10:25:17.705 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-03-27T10:25:17.714 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-03-27T10:25:17.738 INFO:teuthology.task.internal:Duration was 1274.600099 seconds 2023-03-27T10:25:17.738 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-03-27T10:25:17.772 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-03-27T10:25:17.772 DEBUG:teuthology.orchestra.run.smithi159:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-03-27T10:25:17.775 DEBUG:teuthology.orchestra.run.smithi161:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-03-27T10:25:17.777 DEBUG:teuthology.orchestra.run.smithi171:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-03-27T10:25:17.820 INFO:teuthology.orchestra.run.smithi159.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-03-27T10:25:17.823 INFO:teuthology.orchestra.run.smithi171.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-03-27T10:25:17.828 INFO:teuthology.orchestra.run.smithi161.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-03-27T10:25:18.238 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-03-27T10:25:18.238 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi159.front.sepia.ceph.com 2023-03-27T10:25:18.239 DEBUG:teuthology.orchestra.run.smithi159:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-03-27T10:25:18.261 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi161.front.sepia.ceph.com 2023-03-27T10:25:18.261 DEBUG:teuthology.orchestra.run.smithi161:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-03-27T10:25:18.286 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi171.front.sepia.ceph.com 2023-03-27T10:25:18.287 DEBUG:teuthology.orchestra.run.smithi171:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-03-27T10:25:18.309 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-03-27T10:25:18.309 DEBUG:teuthology.orchestra.run.smithi159:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-03-27T10:25:18.311 DEBUG:teuthology.orchestra.run.smithi161:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-03-27T10:25:18.329 DEBUG:teuthology.orchestra.run.smithi171:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-03-27T10:25:18.393 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-03-27T10:25:18.394 DEBUG:teuthology.orchestra.run.smithi159:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-03-27T10:25:18.396 DEBUG:teuthology.orchestra.run.smithi161:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-03-27T10:25:18.398 DEBUG:teuthology.orchestra.run.smithi171:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-03-27T10:25:18.498 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-03-27T10:25:18.512 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-03-27T10:25:18.512 DEBUG:teuthology.orchestra.run.smithi159:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-03-27T10:25:18.539 DEBUG:teuthology.orchestra.run.smithi161:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-03-27T10:25:18.567 DEBUG:teuthology.orchestra.run.smithi171:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-03-27T10:25:18.594 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-03-27T10:25:18.605 DEBUG:teuthology.orchestra.run.smithi159:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-03-27T10:25:18.671 DEBUG:teuthology.orchestra.run.smithi161:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-03-27T10:25:18.673 DEBUG:teuthology.orchestra.run.smithi171:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-03-27T10:25:18.695 INFO:teuthology.orchestra.run.smithi159.stdout:kernel.core_pattern = core 2023-03-27T10:25:18.697 INFO:teuthology.orchestra.run.smithi161.stdout:kernel.core_pattern = core 2023-03-27T10:25:18.699 INFO:teuthology.orchestra.run.smithi171.stdout:kernel.core_pattern = core 2023-03-27T10:25:18.736 DEBUG:teuthology.orchestra.run.smithi159:> test -e /home/ubuntu/cephtest/archive/coredump 2023-03-27T10:25:18.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-03-27T10:25:18.771 DEBUG:teuthology.orchestra.run.smithi161:> test -e /home/ubuntu/cephtest/archive/coredump 2023-03-27T10:25:18.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-03-27T10:25:18.787 DEBUG:teuthology.orchestra.run.smithi171:> test -e /home/ubuntu/cephtest/archive/coredump 2023-03-27T10:25:18.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-03-27T10:25:18.803 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-03-27T10:25:18.813 INFO:teuthology.task.internal:Transferring archived files... 2023-03-27T10:25:18.814 DEBUG:teuthology.misc:Transferring archived files from smithi159:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi/7221317/remote/smithi159 2023-03-27T10:25:18.814 DEBUG:teuthology.orchestra.run.smithi159:> sudo tar cz -f - -C /home/ubuntu/cephtest/archive -- . 2023-03-27T10:25:18.864 DEBUG:teuthology.misc:Transferring archived files from smithi161:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi/7221317/remote/smithi161 2023-03-27T10:25:18.864 DEBUG:teuthology.orchestra.run.smithi161:> sudo tar cz -f - -C /home/ubuntu/cephtest/archive -- . 2023-03-27T10:25:18.912 DEBUG:teuthology.misc:Transferring archived files from smithi171:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/rishabh-2023-03-27_06:39:30-fs-wip-rishabh-cephman-6-testing-default-smithi/7221317/remote/smithi171 2023-03-27T10:25:18.913 DEBUG:teuthology.orchestra.run.smithi171:> sudo tar cz -f - -C /home/ubuntu/cephtest/archive -- . 2023-03-27T10:25:18.961 INFO:teuthology.task.internal:Removing archive directory... 2023-03-27T10:25:18.961 DEBUG:teuthology.orchestra.run.smithi159:> rm -rf -- /home/ubuntu/cephtest/archive 2023-03-27T10:25:18.964 DEBUG:teuthology.orchestra.run.smithi161:> rm -rf -- /home/ubuntu/cephtest/archive 2023-03-27T10:25:18.966 DEBUG:teuthology.orchestra.run.smithi171:> rm -rf -- /home/ubuntu/cephtest/archive 2023-03-27T10:25:19.016 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-03-27T10:25:19.028 INFO:teuthology.task.internal:Not uploading archives. 2023-03-27T10:25:19.028 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-03-27T10:25:19.056 INFO:teuthology.task.internal:Tidying up after the test... 2023-03-27T10:25:19.056 DEBUG:teuthology.orchestra.run.smithi159:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-03-27T10:25:19.058 DEBUG:teuthology.orchestra.run.smithi161:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-03-27T10:25:19.061 DEBUG:teuthology.orchestra.run.smithi171:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-03-27T10:25:19.075 INFO:teuthology.orchestra.run.smithi161.stdout: 262181 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 27 10:25 /home/ubuntu/cephtest 2023-03-27T10:25:19.077 INFO:teuthology.orchestra.run.smithi171.stdout: 262181 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 27 10:25 /home/ubuntu/cephtest 2023-03-27T22:04:44.456 DEBUG:teuthology.exit:Got signal 15; running 1 handler... 2023-03-27T22:04:44.477 DEBUG:teuthology.task.console_log:Killing console logger for smithi159 2023-03-27T22:04:44.478 DEBUG:teuthology.task.console_log:Killing console logger for smithi161 2023-03-27T22:04:44.478 DEBUG:teuthology.task.console_log:Killing console logger for smithi171 2023-03-27T22:04:44.479 DEBUG:teuthology.exit:Finished running handlers